restore npm dependency on iconv-lite
mscdex
11 years ago
parent
41554f5188
commit
7dd577a133
@ -1,3 +0,0 @@
|
||||
node_modules
|
||||
*~
|
||||
*sublime-*
|
||||
@ -1,5 +0,0 @@
|
||||
language: node_js
|
||||
node_js:
|
||||
- 0.4
|
||||
- 0.6
|
||||
- 0.8
|
||||
@ -1,21 +0,0 @@
|
||||
Copyright (c) 2011 Alexander Shtuchkin
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
||||
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
||||
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
var big5Table = require('./table/big5.js');
|
||||
module.exports = {
|
||||
'windows950': 'big5',
|
||||
'cp950': 'big5',
|
||||
'big5': {
|
||||
type: 'table',
|
||||
table: big5Table
|
||||
}
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
var gbkTable = require('./table/gbk.js');
|
||||
module.exports = {
|
||||
'windows936': 'gbk',
|
||||
'gb2312': 'gbk',
|
||||
'gbk': {
|
||||
type: 'table',
|
||||
table: gbkTable
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -1,25 +0,0 @@
|
||||
var http = require('http');
|
||||
var fs = require('fs');
|
||||
// BIG5
|
||||
var cp950_b2u = {host:'moztw.org',path:'/docs/big5/table/cp950-b2u.txt'},
|
||||
cp950_u2b = {host:'moztw.org',path:'/docs/big5/table/cp950-u2b.txt'},
|
||||
cp950_moz18_b2u = {host:'moztw.org',path:'/docs/big5/table/moz18-b2u.txt'};
|
||||
|
||||
http.get(cp950_moz18_b2u, function(res) {
|
||||
var data = '';
|
||||
res.on('data', function(chunk) {
|
||||
data += chunk;
|
||||
});
|
||||
res.on('end', function() {
|
||||
var table = {};
|
||||
data = data.split('\n').slice(1);
|
||||
data.forEach(function(line, idx) {
|
||||
var pair = line.split(' ');
|
||||
var key = parseInt(pair[0]);
|
||||
var val = parseInt(pair[1]);
|
||||
table[key] = val;
|
||||
});
|
||||
fs.createWriteSync('encodings/table/big5.js',
|
||||
'module.exports = ' + JSON.stringify(table) + ';');
|
||||
});
|
||||
});
|
||||
@ -1,142 +0,0 @@
|
||||
var fs = require("fs");
|
||||
var Iconv = require("iconv").Iconv;
|
||||
|
||||
|
||||
var encodingFamilies = [
|
||||
{
|
||||
// Windows code pages
|
||||
encodings: [1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258],
|
||||
convert: function(cp) {
|
||||
return {
|
||||
name: "windows-"+cp,
|
||||
aliases: ["win"+cp, "cp"+cp, ""+cp],
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
// ISO-8859 code pages
|
||||
encodings: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16],
|
||||
convert: function(i) {
|
||||
return {
|
||||
name: "iso-8859-"+i,
|
||||
aliases: ["cp"+(28590+i), (28590+i)],
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
// IBM/DOS code pages
|
||||
encodings: [437, 737, 775, 850, 852, 855, 857, 858, 860, 861, 862, 863, 864, 865, 866, 869],
|
||||
convert: function(cp) {
|
||||
return {
|
||||
name: "CP"+cp,
|
||||
aliases: ["ibm"+cp, ""+cp],
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
// Macintosh code pages
|
||||
encodings: ["macCroatian", "macCyrillic", "macGreek",
|
||||
"macIceland", "macRoman", "macRomania",
|
||||
"macThai", "macTurkish", "macUkraine"],
|
||||
},
|
||||
{
|
||||
// KOI8 code pages
|
||||
encodings: ["KOI8-R", "KOI8-U"],
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
var encodings = {
|
||||
// Aliases.
|
||||
"ascii8bit": "ascii",
|
||||
"usascii": "ascii",
|
||||
|
||||
"latin1": "iso88591",
|
||||
"latin2": "iso88592",
|
||||
"latin3": "iso88593",
|
||||
"latin4": "iso88594",
|
||||
"latin6": "iso885910",
|
||||
"latin7": "iso885913",
|
||||
"latin8": "iso885914",
|
||||
"latin9": "iso885915",
|
||||
"latin10": "iso885916",
|
||||
|
||||
"cp819": "iso88951",
|
||||
"arabic": "iso88596",
|
||||
"arabic8": "iso88596",
|
||||
"greek" : "iso88597",
|
||||
"greek8" : "iso88597",
|
||||
"hebrew": "iso88598",
|
||||
"hebrew8": "iso88598",
|
||||
"turkish": "iso88599",
|
||||
"turkish8": "iso88599",
|
||||
"thai": "iso885911",
|
||||
"thai8": "iso885911",
|
||||
"tis620": "iso885911",
|
||||
"windows874": "iso885911",
|
||||
"win874": "iso885911",
|
||||
"cp874": "iso885911",
|
||||
"874": "iso885911",
|
||||
"celtic": "iso885914",
|
||||
"celtic8": "iso885914",
|
||||
|
||||
"cp20866": "koi8r",
|
||||
"20866": "koi8r",
|
||||
"ibm878": "koi8r",
|
||||
"cp21866": "koi8u",
|
||||
"21866": "koi8u",
|
||||
"ibm1168": "koi8u",
|
||||
|
||||
};
|
||||
|
||||
// Add all encodings from encodingFamilies.
|
||||
encodingFamilies.forEach(function(family){
|
||||
family.encodings.forEach(function(encoding){
|
||||
if (family.convert)
|
||||
encoding = family.convert(encoding);
|
||||
|
||||
var encodingIconvName = encoding.name ? encoding.name : encoding;
|
||||
var encodingName = encodingIconvName.replace(/[-_]/g, "").toLowerCase();
|
||||
|
||||
encodings[encodingName] = {
|
||||
type: "singlebyte",
|
||||
chars: generateCharsString(encodingIconvName)
|
||||
};
|
||||
|
||||
if (encoding.aliases)
|
||||
encoding.aliases.forEach(function(alias){
|
||||
encodings[alias] = encodingName;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Write encodings.
|
||||
fs.writeFileSync("encodings/singlebyte.js",
|
||||
"module.exports = " + JSON.stringify(encodings, undefined, " ") + ";");
|
||||
|
||||
|
||||
function generateCharsString(encoding) {
|
||||
console.log("Generate encoding for " + encoding);
|
||||
var iconvToUtf8 = new Iconv(encoding, "UTF-8");
|
||||
var chars = "";
|
||||
|
||||
for (var b = 0x80; b < 0x100; b++) {
|
||||
|
||||
try {
|
||||
var convertedChar = iconvToUtf8.convert(new Buffer([b])).toString();
|
||||
|
||||
if (convertedChar.length != 1)
|
||||
throw new Error("Single-byte encoding error: Must return single char.");
|
||||
} catch (exception) {
|
||||
if (exception.code === "EILSEQ") {
|
||||
convertedChar = "\ufffd";
|
||||
} else {
|
||||
throw exception;
|
||||
}
|
||||
}
|
||||
|
||||
chars += convertedChar;
|
||||
}
|
||||
|
||||
return chars;
|
||||
}
|
||||
@ -1,13 +0,0 @@
|
||||
<HTML>
|
||||
<HEAD>
|
||||
<TITLE> meta 標籤的使用：中文網頁 </TITLE>
|
||||
<meta HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=big5">
|
||||
</HEAD>
|
||||
<BODY>
|
||||
|
||||
這是一個繁體中文網頁！<br>
|
||||
(This page uses big5 character set.)<br>
|
||||
charset=big5
|
||||
|
||||
</BODY>
|
||||
</HTML>
|
||||
@ -1,79 +0,0 @@
|
||||
var vows = require('vows'),
|
||||
assert = require('assert'),
|
||||
iconv = require(__dirname+'/../');
|
||||
|
||||
var baseStrings = {
|
||||
empty: "",
|
||||
hi: "Γειά!",
|
||||
ascii: '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f'+
|
||||
' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f',
|
||||
greek: "αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩάέήίόύώΆΈΉΊΌΎΏϊϋΪΫ",
|
||||
untranslatable: "Åçþÿ¿"
|
||||
};
|
||||
|
||||
var encodings = [{
|
||||
name: "windows1253",
|
||||
variations: ['windows-1253', 'win-1253', 'win1253', 'cp1253', 'cp-1253', 1253],
|
||||
encodedStrings: {
|
||||
empty: new Buffer(''),
|
||||
hi: new Buffer('\xc3\xe5\xe9\xdc!', 'binary'),
|
||||
ascii: new Buffer(baseStrings.ascii, 'binary'),
|
||||
greek: new Buffer('\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xdc\xdd\xde\xdf\xfc\xfd\xfe\xa2\xb8\xb9\xba\xbc\xbe\xbf\xfa\xfb\xda\xdb', 'binary'),
|
||||
}
|
||||
}, {
|
||||
name: "iso88597",
|
||||
variations: ['iso-8859-7', 'greek', 'greek8', 'cp28597', 'cp-28597', 28597],
|
||||
encodedStrings: {
|
||||
empty: new Buffer(''),
|
||||
hi: new Buffer('\xc3\xe5\xe9\xdc!', 'binary'),
|
||||
ascii: new Buffer(baseStrings.ascii, 'binary'),
|
||||
greek: new Buffer('\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xdc\xdd\xde\xdf\xfc\xfd\xfe\xb6\xb8\xb9\xba\xbc\xbe\xbf\xfa\xfb\xda\xdb', 'binary'),
|
||||
}
|
||||
}, {
|
||||
name: "cp737",
|
||||
variations: ['cp-737', 737],
|
||||
encodedStrings: {
|
||||
empty: new Buffer(''),
|
||||
hi: new Buffer('\x82\x9c\xa0\xe1!', 'binary'),
|
||||
ascii: new Buffer(baseStrings.ascii, 'binary'),
|
||||
greek: new Buffer('\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xab\xac\xad\xae\xaf\xe0\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\xe1\xe2\xe3\xe5\xe6\xe7\xe9\xea\xeb\xec\xed\xee\xef\xf0\xe4\xe8\xf4\xf5', 'binary'),
|
||||
}
|
||||
}];
|
||||
|
||||
var testsBatch = {};
|
||||
encodings.forEach(function(encoding) {
|
||||
var enc = encoding.variations[0];
|
||||
var key = "hi";
|
||||
var tests = {
|
||||
"Convert to empty buffer": function() {
|
||||
assert.strictEqual(iconv.toEncoding("", enc).toString('binary'), new Buffer('').toString('binary'));
|
||||
},
|
||||
"Convert from empty buffer": function() {
|
||||
assert.strictEqual(iconv.fromEncoding(new Buffer(''), enc), "");
|
||||
},
|
||||
"Convert from buffer": function() {
|
||||
for (var key in encoding.encodedStrings)
|
||||
assert.strictEqual(iconv.fromEncoding(encoding.encodedStrings[key], enc),
|
||||
baseStrings[key]);
|
||||
},
|
||||
"Convert to buffer": function() {
|
||||
for (var key in encoding.encodedStrings)
|
||||
assert.strictEqual(iconv.toEncoding(baseStrings[key], enc).toString('binary'),
|
||||
encoding.encodedStrings[key].toString('binary'));
|
||||
},
|
||||
"Try different variations of encoding": function() {
|
||||
encoding.variations.forEach(function(enc) {
|
||||
assert.strictEqual(iconv.fromEncoding(encoding.encodedStrings[key], enc), baseStrings[key]);
|
||||
assert.strictEqual(iconv.toEncoding(baseStrings[key], enc).toString('binary'), encoding.encodedStrings[key].toString('binary'));
|
||||
});
|
||||
},
|
||||
"Untranslatable chars are converted to defaultCharSingleByte": function() {
|
||||
var expected = baseStrings.untranslatable.split('').map(function(c) {return iconv.defaultCharSingleByte; }).join('');
|
||||
assert.strictEqual(iconv.toEncoding(baseStrings.untranslatable, enc).toString('binary'), expected); // Only '?' characters.
|
||||
}
|
||||
};
|
||||
|
||||
testsBatch[encoding.name+":"] = tests;
|
||||
});
|
||||
|
||||
vows.describe("Test Greek encodings").addBatch(testsBatch).export(module);
|
||||
@ -1,55 +0,0 @@
|
||||
var vows = require('vows'),
|
||||
assert = require('assert'),
|
||||
iconv = require(__dirname+'/../');
|
||||
|
||||
var testString = "Hello123!";
|
||||
var testStringLatin1 = "Hello123!£Å÷×çþÿ¿®";
|
||||
var testStringBase64 = "SGVsbG8xMjMh";
|
||||
|
||||
vows.describe("Generic UTF8-UCS2 tests").addBatch({
|
||||
"Vows is working": function() {},
|
||||
"Return values are of correct types": function() {
|
||||
assert.ok(iconv.toEncoding(testString, "utf8") instanceof Buffer);
|
||||
|
||||
var s = iconv.fromEncoding(new Buffer(testString), "utf8");
|
||||
assert.strictEqual(Object.prototype.toString.call(s), "[object String]");
|
||||
},
|
||||
"Internal encodings all correctly encoded/decoded": function() {
|
||||
['utf8', "UTF-8", "UCS2", "binary", ""].forEach(function(enc) {
|
||||
assert.strictEqual(iconv.toEncoding(testStringLatin1, enc).toString(enc), testStringLatin1);
|
||||
assert.strictEqual(iconv.fromEncoding(new Buffer(testStringLatin1, enc), enc), testStringLatin1);
|
||||
});
|
||||
},
|
||||
"Base64 correctly encoded/decoded": function() {
|
||||
assert.strictEqual(iconv.toEncoding(testStringBase64, "base64").toString("binary"), testString);
|
||||
assert.strictEqual(iconv.fromEncoding(new Buffer(testString, "binary"), "base64"), testStringBase64);
|
||||
},
|
||||
"Latin1 correctly encoded/decoded": function() {
|
||||
assert.strictEqual(iconv.toEncoding(testStringLatin1, "latin1").toString("binary"), testStringLatin1);
|
||||
assert.strictEqual(iconv.fromEncoding(new Buffer(testStringLatin1, "binary"), "latin1"), testStringLatin1);
|
||||
},
|
||||
"Convert from string, not buffer (binary encoding used)": function() {
|
||||
assert.strictEqual(iconv.fromEncoding(testStringLatin1, "binary"), testStringLatin1);
|
||||
},
|
||||
"Convert to string, not buffer (utf8 used)": function() {
|
||||
var res = iconv.toEncoding(new Buffer(testStringLatin1, "utf8"));
|
||||
assert.ok(res instanceof Buffer);
|
||||
assert.strictEqual(res.toString("utf8"), testStringLatin1);
|
||||
},
|
||||
"Throws on unknown encodings": function() {
|
||||
assert.throws(function() { iconv.toEncoding("a", "xxx"); });
|
||||
assert.throws(function() { iconv.fromEncoding("a", "xxx"); });
|
||||
},
|
||||
"Convert non-strings and non-buffers": function() {
|
||||
assert.strictEqual(iconv.toEncoding({}, "utf8").toString(), "[object Object]");
|
||||
assert.strictEqual(iconv.toEncoding(10, "utf8").toString(), "10");
|
||||
assert.strictEqual(iconv.toEncoding(undefined, "utf8").toString(), "");
|
||||
assert.strictEqual(iconv.fromEncoding({}, "utf8"), "[object Object]");
|
||||
assert.strictEqual(iconv.fromEncoding(10, "utf8"), "10");
|
||||
assert.strictEqual(iconv.fromEncoding(undefined, "utf8"), "");
|
||||
},
|
||||
"Aliases encode and decode work the same as toEncoding and fromEncoding": function() {
|
||||
assert.strictEqual(iconv.toEncoding(testString, "latin1").toString("binary"), iconv.encode(testString, "latin1").toString("binary"));
|
||||
assert.strictEqual(iconv.fromEncoding(testStringLatin1, "latin1"), iconv.decode(testStringLatin1, "latin1"));
|
||||
},
|
||||
}).export(module)
|
||||
@ -1,67 +0,0 @@
|
||||
|
||||
var iconv = require('iconv');
|
||||
var iconv_lite = require("../index");
|
||||
|
||||
var encoding = process.argv[2] || "windows-1251";
|
||||
var convertTimes = 10000;
|
||||
|
||||
var encodingStrings = {
|
||||
'windows-1251': 'This is a test string 32 chars..',
|
||||
'gbk': '这是中文字符测试。。！@￥%12',
|
||||
'utf8': '这是中文字符测试。。！@￥%12This is a test string 48 chars..',
|
||||
};
|
||||
// Test encoding.
|
||||
var str = encodingStrings[encoding];
|
||||
if (!str) {
|
||||
throw new Error('Don\'t support ' + encoding + ' performance test.');
|
||||
}
|
||||
for (var i = 0; i < 13; i++) {
|
||||
str = str + str;
|
||||
}
|
||||
|
||||
console.log('\n' + encoding + ' charset performance test:');
|
||||
console.log("\nEncoding "+str.length+" chars "+convertTimes+" times:");
|
||||
|
||||
var start = Date.now();
|
||||
var converter = new iconv.Iconv("utf8", encoding);
|
||||
for (var i = 0; i < convertTimes; i++) {
|
||||
var b = converter.convert(str);
|
||||
}
|
||||
var duration = Date.now() - start;
|
||||
var mbs = convertTimes*b.length/duration/1024;
|
||||
|
||||
console.log("iconv: "+duration+"ms, "+mbs.toFixed(2)+" Mb/s.");
|
||||
|
||||
var start = Date.now();
|
||||
for (var i = 0; i < convertTimes; i++) {
|
||||
var b = iconv_lite.encode(str, encoding);
|
||||
}
|
||||
var duration = Date.now() - start;
|
||||
var mbs = convertTimes*b.length/duration/1024;
|
||||
|
||||
console.log("iconv-lite: "+duration+"ms, "+mbs.toFixed(2)+" Mb/s.");
|
||||
|
||||
|
||||
// Test decoding.
|
||||
var buf = iconv_lite.encode(str, encoding);
|
||||
console.log("\nDecoding "+buf.length+" bytes "+convertTimes+" times:");
|
||||
|
||||
var start = Date.now();
|
||||
var converter = new iconv.Iconv(encoding, "utf8");
|
||||
for (var i = 0; i < convertTimes; i++) {
|
||||
var s = converter.convert(buf).toString();
|
||||
}
|
||||
var duration = Date.now() - start;
|
||||
var mbs = convertTimes*buf.length/duration/1024;
|
||||
|
||||
console.log("iconv: "+duration+"ms, "+mbs.toFixed(2)+" Mb/s.");
|
||||
|
||||
var start = Date.now();
|
||||
for (var i = 0; i < convertTimes; i++) {
|
||||
var s = iconv_lite.decode(buf, encoding);
|
||||
}
|
||||
var duration = Date.now() - start;
|
||||
var mbs = convertTimes*buf.length/duration/1024;
|
||||
|
||||
console.log("iconv-lite: "+duration+"ms, "+mbs.toFixed(2)+" Mb/s.");
|
||||
|
||||
Loading…
Reference in New Issue