Skip to content

Commit 457f8ab

Browse files
SheetJSDevSheetJSDev
committed
version bump 1.3.4: more codepages
- added codepage 21027 (generated from NLS) - unicode decoders skip BOM - codepage command line tool -B writes bom - version number added to scripts - codepages 1257/1258 added to cpexcel
1 parent 8bb326d commit 457f8ab

22 files changed

+460
-58
lines changed

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ hardcoded in utils, there is no corresponding entry (they are "magic")
199199
|20936|MakeEncoding.cs|Simplified Chinese (GB2312-80)
200200
|20949|MakeEncoding.cs|Korean Wansung
201201
|21025|MakeEncoding.cs|IBM EBCDIC Cyrillic Serbian-Bulgarian
202+
|21027| NLS |Extended/Ext Alpha Lowercase
202203
|21866|MakeEncoding.cs|Ukrainian Cyrillic (KOI8-U)
203204
|28591| unicode.org |ISO 8859-1 Latin 1 (Western European)
204205
|28592| unicode.org |ISO 8859-2 Latin 2 (Central European)
@@ -254,7 +255,6 @@ ISO-8869-6 when in fact there are many differences), so all implementations
254255

255256
- 709 Arabic (ASMO-449+, BCON V4)
256257
- 710 Arabic - Transparent Arabic
257-
- 21027 (deprecated) <-- is this necessary?
258258
- 50229 ISO 2022 Traditional Chinese
259259
- 50930 EBCDIC Japanese (Katakana) Extended
260260
- 50931 EBCDIC US-Canada and Japanese

‎bin/codepage.njs

+24-4
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@
44
var codepage = require('../');
55
var fs = require('fs'), program = require('commander');
66
program
7-
.version('1.0.0')
7+
.version(codepage.version)
88
.usage('[options] <file>')
99
.option('-f, --from-code <code>', 'codepage of input (default 65001 utf8)')
1010
.option('-t, --to-code <code>', 'codepage of output (default 65001 utf8)')
1111
.option('-o, --output <file>', 'output file (<file>.<to> if specified)')
12+
.option('-B, --bom', 'write BOM (for unicode codepages)')
1213
.option('-l, --list', 'List supported codepages');
1314

1415
program.on('--help', function() {
@@ -19,6 +20,11 @@ program.on('--help', function() {
1920

2021
program.parse(process.argv);
2122

23+
if(program.list) {
24+
Object.keys(codepage).forEach(function(x) { if(+x == x) console.log(x); });
25+
process.exit();
26+
}
27+
2228
if(!fs.existsSync(program.args[0])) {
2329
console.error('codepage: must specify a filename');
2430
process.exit(13);
@@ -30,6 +36,20 @@ var f = program.args[0];
3036
var o = program.output;
3137
var text = fs.readFileSync(f);
3238
var dec = codepage.utils.decode(fr, text);
33-
if(o) fs.writeFileSync(o, codepage.utils.encode(to, dec));
34-
else if(!program.toCode) console.log(dec.toString('utf8'));
35-
else fs.writeFileSync(f + "." + to, codepage.utils.encode(to, dec));
39+
40+
var bom = {
41+
1200: new Buffer([0xFF, 0xFE]),
42+
1201: new Buffer([0xFE, 0xFF]),
43+
12000: new Buffer([0xFF, 0xFE, 0x00, 0x00]),
44+
12001: new Buffer([0x00, 0x00, 0xFE, 0xFF]),
45+
16969: new Buffer([0x69, 0x69]),
46+
65000: new Buffer([0x2B, 0x2F, 0x76, 0x2B]),
47+
65001: new Buffer([0xEF, 0xBB, 0xBF])
48+
}
49+
50+
if(!program.toCode && !o) console.log(dec.toString('utf8'));
51+
else if(!program.bom || !bom[fr]) fs.writeFileSync(o || (f + "." + to), codepage.utils.encode(to, dec));
52+
else {
53+
fs.writeFileSync(o || (f + "." + to), bom[fr]);
54+
fs.appendFileSync(o || (f + "." + to), codepage.utils.encode(to, dec));
55+
}

‎bits/21027.js

+2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎codepage.md

+7-4
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ The following codepages are available in .NET on Windows:
137137
- 20936 Simplified Chinese (GB2312); Chinese Simplified (GB2312-80)
138138
- 20949 Korean Wansung
139139
- 21025 IBM EBCDIC Cyrillic Serbian-Bulgarian
140+
- 21027 Extended/Ext Alpha Lowercase
140141
- 21866 Ukrainian (KOI8-U); Cyrillic (KOI8-U)
141142
- 29001 Europa 3
142143
- 38598 ISO 8859-8 Hebrew; Hebrew (ISO-Logical)
@@ -222,6 +223,7 @@ The following codepages are available in .NET on Windows:
222223
20936,,2
223224
20949,,2
224225
21025,,1
226+
21027,,1
225227
21866,,1
226228
29001,,1
227229
38598,,1
@@ -389,12 +391,13 @@ elements (like `0x7F` for CP 10000) are removed.
389391
INFILE=${1:-pages.csv}
390392
OUTFILE=${2:-cptable.js}
391393
JSVAR=${3:-cptable}
394+
VERSION=$(cat package.json | grep version | tr -dc [0-9.])
392395

393396
mkdir -p codepages bits
394397
rm -f $OUTFILE $OUTFILE.tmp
395398
echo "/* $OUTFILE (C) 2013-2014 SheetJS -- http://sheetjs.com */" > $OUTFILE.tmp
396399
echo "/*jshint -W100 */" >> $OUTFILE.tmp
397-
echo "var $JSVAR = {};" >> $OUTFILE.tmp
400+
echo "var $JSVAR = {version:\"$VERSION\"};" >> $OUTFILE.tmp
398401
if [ -e dotnet.sh ]; then bash dotnet.sh; fi
399402
awk -F, '{print $1, $2, $3}' $INFILE | while read cp url cptype; do
400403
echo $cp $url
@@ -510,8 +513,8 @@ describe('consistency', function() {
510513
cptable.utils.cache.encache();
511514
});
512515
}; };
513-
Object.keys(cptable).filter(function(w) { return w != "utils"; }).forEach(chk(cptable, true));
514-
Object.keys(cptable).filter(function(w) { return w != "utils"; }).forEach(chk(cptable, false));
516+
Object.keys(cptable).filter(function(w) { return w == +w; }).forEach(chk(cptable, true));
517+
Object.keys(cptable).filter(function(w) { return w == +w; }).forEach(chk(cptable, false));
515518
});
516519
```
517520

@@ -712,7 +715,7 @@ describe('failures', function() {
712715
```json>package.json
713716
{
714717
"name": "codepage",
715-
"version": "1.3.3",
718+
"version": "1.3.4",
716719
"author": "SheetJS",
717720
"description": "pure-JS library to handle codepages",
718721
"keywords": [ "codepage", "iconv", "convert", "strings" ],

‎codepages/21027.TBL

+256
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
0x00 0x0000
2+
0x01 0x0001
3+
0x02 0x0002
4+
0x03 0x0003
5+
0x04 0x0004
6+
0x05 0x0005
7+
0x06 0x0006
8+
0x07 0x0007
9+
0x08 0x0008
10+
0x09 0x0009
11+
0x0A 0x000A
12+
0x0B 0x000B
13+
0x0C 0x000C
14+
0x0D 0x000D
15+
0x0E 0x000E
16+
0x0F 0x000F
17+
0x10 0x0010
18+
0x11 0x0011
19+
0x12 0x0012
20+
0x13 0x0013
21+
0x14 0x0014
22+
0x15 0x0015
23+
0x16 0x0016
24+
0x17 0x0017
25+
0x18 0x0018
26+
0x19 0x0019
27+
0x1A 0x001A
28+
0x1B 0x001B
29+
0x1C 0x001C
30+
0x1D 0x001D
31+
0x1E 0x001E
32+
0x1F 0x001F
33+
0x20 0xF8C4
34+
0x21 0xF8C5
35+
0x22 0xF8C6
36+
0x23 0xF8C7
37+
0x24 0xF8C8
38+
0x25 0xF8C9
39+
0x26 0xF8CA
40+
0x27 0xF8CB
41+
0x28 0xF8CC
42+
0x29 0xF8CD
43+
0x2A 0xF8CE
44+
0x2B 0xF8CF
45+
0x2C 0xF8D0
46+
0x2D 0xF8D1
47+
0x2E 0xF8D2
48+
0x2F 0xF8D3
49+
0x30 0xF8D4
50+
0x31 0xF8D5
51+
0x32 0xF8D6
52+
0x33 0xF8D7
53+
0x34 0xF8D8
54+
0x35 0xF8D9
55+
0x36 0xF8DA
56+
0x37 0xF8DB
57+
0x38 0xF8DC
58+
0x39 0xF8DD
59+
0x3A 0xF8DE
60+
0x3B 0xF8DF
61+
0x3C 0xF8E0
62+
0x3D 0xF8E1
63+
0x3E 0x0000
64+
0x3F 0xF8E2
65+
0x40 0x0020
66+
0x41 0x0000
67+
0x42 0xFF61
68+
0x43 0xFF62
69+
0x44 0xFF63
70+
0x45 0xFF64
71+
0x46 0xFF65
72+
0x47 0xFF66
73+
0x48 0xFF67
74+
0x49 0xFF68
75+
0x4A 0x00A2
76+
0x4B 0x002E
77+
0x4C 0x003C
78+
0x4D 0x0028
79+
0x4E 0x002B
80+
0x4F 0x007C
81+
0x50 0x0026
82+
0x51 0xFF69
83+
0x52 0xFF6A
84+
0x53 0xFF6B
85+
0x54 0xFF6C
86+
0x55 0xFF6D
87+
0x56 0xFF6E
88+
0x57 0xFF6F
89+
0x58 0xFF70
90+
0x59 0xFF71
91+
0x5A 0x0021
92+
0x5B 0x0024
93+
0x5C 0x002A
94+
0x5D 0x0029
95+
0x5E 0x003B
96+
0x5F 0x00AC
97+
0x60 0x002D
98+
0x61 0x002F
99+
0x62 0xFF72
100+
0x63 0xFF73
101+
0x64 0xFF74
102+
0x65 0xFF75
103+
0x66 0xFF76
104+
0x67 0xFF77
105+
0x68 0xF8E3
106+
0x69 0xFF79
107+
0x6A 0x0000
108+
0x6B 0x002C
109+
0x6C 0x0025
110+
0x6D 0x005F
111+
0x6E 0x003E
112+
0x6F 0x003F
113+
0x70 0xFF7A
114+
0x71 0xFF7B
115+
0x72 0xFF7C
116+
0x73 0xFF7D
117+
0x74 0xFF7E
118+
0x75 0xFF7F
119+
0x76 0xFF80
120+
0x77 0xFF81
121+
0x78 0xFF82
122+
0x79 0x0060
123+
0x7A 0x003A
124+
0x7B 0x0023
125+
0x7C 0x0040
126+
0x7D 0x0027
127+
0x7E 0xF8E4
128+
0x7F 0x0022
129+
0x80 0x0000
130+
0x81 0x0061
131+
0x82 0x0062
132+
0x83 0x0063
133+
0x84 0x0064
134+
0x85 0x0065
135+
0x86 0x0066
136+
0x87 0x0067
137+
0x88 0x0068
138+
0x89 0x0069
139+
0x8A 0xFF83
140+
0x8B 0xFF84
141+
0x8C 0xFF85
142+
0x8D 0xFF86
143+
0x8E 0x008E
144+
0x8F 0xFF88
145+
0x90 0x0000
146+
0x91 0x006A
147+
0x92 0x006B
148+
0x93 0x006C
149+
0x94 0x006D
150+
0x95 0x006E
151+
0x96 0x006F
152+
0x97 0x0070
153+
0x98 0x0071
154+
0x99 0x0072
155+
0x9A 0xFF89
156+
0x9B 0xFF8A
157+
0x9C 0xFF8B
158+
0x9D 0xFF8C
159+
0x9E 0xFF8D
160+
0x9F 0xFF8E
161+
0xA0 0x00AF
162+
0xA1 0x007E
163+
0xA2 0x0073
164+
0xA3 0x0074
165+
0xA4 0x0075
166+
0xA5 0x0076
167+
0xA6 0x0077
168+
0xA7 0x0078
169+
0xA8 0x0079
170+
0xA9 0x007A
171+
0xAA 0xFF8F
172+
0xAB 0xFF90
173+
0xAC 0xFF91
174+
0xAD 0x005B
175+
0xAE 0xFF92
176+
0xAF 0xFF93
177+
0xB0 0x005E
178+
0xB1 0x00A3
179+
0xB2 0x00A5
180+
0xB3 0xFF94
181+
0xB4 0xFF95
182+
0xB5 0xFF96
183+
0xB6 0xFF97
184+
0xB7 0xFF98
185+
0xB8 0xFF99
186+
0xB9 0xFF9A
187+
0xBA 0xFF9B
188+
0xBB 0xFF9C
189+
0xBC 0xFF9D
190+
0xBD 0x005D
191+
0xBE 0xFF9E
192+
0xBF 0xFF9F
193+
0xC0 0x007B
194+
0xC1 0x0041
195+
0xC2 0x0042
196+
0xC3 0x0043
197+
0xC4 0x0044
198+
0xC5 0x0045
199+
0xC6 0x0046
200+
0xC7 0x0047
201+
0xC8 0xF8E5
202+
0xC9 0xF8E6
203+
0xCA 0xF8E7
204+
0xCB 0xF8E8
205+
0xCC 0xF8E9
206+
0xCD 0xF8EA
207+
0xCE 0x0000
208+
0xCF 0x0000
209+
0xD0 0x007D
210+
0xD1 0x004A
211+
0xD2 0x004B
212+
0xD3 0x004C
213+
0xD4 0x004D
214+
0xD5 0x004E
215+
0xD6 0x004F
216+
0xD7 0x0050
217+
0xD8 0xF8EB
218+
0xD9 0xF8EC
219+
0xDA 0xF8ED
220+
0xDB 0xF8EE
221+
0xDC 0xF8EF
222+
0xDD 0xF8F0
223+
0xDE 0x0000
224+
0xDF 0xF8F1
225+
0xE0 0x005C
226+
0xE1 0x0000
227+
0xE2 0x0053
228+
0xE3 0x0054
229+
0xE4 0x0055
230+
0xE5 0x0056
231+
0xE6 0x0057
232+
0xE7 0x0058
233+
0xE8 0xF8F2
234+
0xE9 0xF8F3
235+
0xEA 0xF8F4
236+
0xEB 0xF8F5
237+
0xEC 0xF8F6
238+
0xED 0xF8F7
239+
0xEE 0x0000
240+
0xEF 0x0000
241+
0xF0 0x0030
242+
0xF1 0x0031
243+
0xF2 0x0032
244+
0xF3 0x0033
245+
0xF4 0x0034
246+
0xF5 0x0035
247+
0xF6 0x0036
248+
0xF7 0x0037
249+
0xF8 0xF8F8
250+
0xF9 0xF8F9
251+
0xFA 0xF8FA
252+
0xFB 0xF8FB
253+
0xFC 0xF8FC
254+
0xFD 0xF8FD
255+
0xFE 0xF8FE
256+
0xFF 0xF8FF

‎cpexcel.js

+3-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)