Skip to content

Commit 26fa66e

Browse files
n1474335n1474335
committed
Merge branch 'newMagic' of https://github.com/n1073645/CyberChef into n1073645-newMagic
2 parents 26b1935 + 5b6a53b commit 26fa66e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+907
-408
lines changed

‎src/core/Utils.mjs

+1
Original file line numberDiff line numberDiff line change
@@ -1182,6 +1182,7 @@ class Utils {
11821182
"CRLF": /\r\n/g,
11831183
"Forward slash": /\//g,
11841184
"Backslash": /\\/g,
1185+
"0x with comma": /,?0x/g,
11851186
"0x": /0x/g,
11861187
"\\x": /\\x/g,
11871188
"None": /\s+/g // Included here to remove whitespace when there shouldn't be any

‎src/core/config/scripts/generateConfig.mjs

+23-4
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,32 @@ for (const opObj in Ops) {
4242
outputType: op.presentType,
4343
flowControl: op.flowControl,
4444
manualBake: op.manualBake,
45-
args: op.args
45+
args: op.args,
4646
};
4747

48-
if ("patterns" in op) {
49-
operationConfig[op.name].patterns = op.patterns;
48+
if ("checks" in op) {
49+
if ("input" in op.checks) {
50+
operationConfig[op.name].input = {};
51+
if ("regex" in op.checks.input) {
52+
operationConfig[op.name].input.regex = op.checks.input.regex;
53+
}
54+
if ("entropy" in op.checks.input) {
55+
operationConfig[op.name].input.entropy = op.checks.input.entropy;
56+
}
57+
}
58+
if ("output" in op.checks) {
59+
operationConfig[op.name].output = {};
60+
if ("regex" in op.checks.output) {
61+
operationConfig[op.name].output.regex = op.checks.output.regex;
62+
}
63+
if ("entropy" in op.checks.output) {
64+
operationConfig[op.name].output.entropy = op.checks.output.entropy;
65+
}
66+
if ("mime" in op.checks.output) {
67+
operationConfig[op.name].output.mime = op.checks.output.mime;
68+
}
69+
}
5070
}
51-
5271
if (!(op.module in modules))
5372
modules[op.module] = {};
5473
modules[op.module][op.name] = opObj;

‎src/core/lib/IP.mjs

+4-4
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ export function ipv4CidrRange(cidr, includeNetworkInfo, enumerateAddresses, allo
2626
let output = "";
2727

2828
if (cidrRange < 0 || cidrRange > 31) {
29-
return "IPv4 CIDR must be less than 32";
29+
throw new OperationError("IPv4 CIDR must be less than 32");
3030
}
3131

3232
const mask = ~(0xFFFFFFFF >>> cidrRange),
@@ -64,7 +64,7 @@ export function ipv6CidrRange(cidr, includeNetworkInfo) {
6464
cidrRange = parseInt(cidr[cidr.length-1], 10);
6565

6666
if (cidrRange < 0 || cidrRange > 127) {
67-
return "IPv6 CIDR must be less than 128";
67+
throw new OperationError("IPv6 CIDR must be less than 128");
6868
}
6969

7070
const ip1 = new Array(8),
@@ -211,7 +211,7 @@ export function ipv4ListedRange(match, includeNetworkInfo, enumerateAddresses, a
211211
const network = strToIpv4(ipv4CidrList[i].split("/")[0]);
212212
const cidrRange = parseInt(ipv4CidrList[i].split("/")[1], 10);
213213
if (cidrRange < 0 || cidrRange > 31) {
214-
return "IPv4 CIDR must be less than 32";
214+
throw new OperationError("IPv4 CIDR must be less than 32");
215215
}
216216
const mask = ~(0xFFFFFFFF >>> cidrRange),
217217
cidrIp1 = network & mask,
@@ -254,7 +254,7 @@ export function ipv6ListedRange(match, includeNetworkInfo) {
254254
const cidrRange = parseInt(ipv6CidrList[i].split("/")[1], 10);
255255

256256
if (cidrRange < 0 || cidrRange > 127) {
257-
return "IPv6 CIDR must be less than 128";
257+
throw new OperationError("IPv6 CIDR must be less than 128");
258258
}
259259

260260
const cidrIp1 = new Array(8),

‎src/core/lib/Magic.mjs

+110-31
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import OperationConfig from "../config/OperationConfig.json";
22
import Utils, { isWorkerEnvironment } from "../Utils.mjs";
33
import Recipe from "../Recipe.mjs";
44
import Dish from "../Dish.mjs";
5-
import {detectFileType} from "./FileType.mjs";
5+
import {detectFileType, isType} from "./FileType.mjs";
66
import chiSquared from "chi-squared";
77

88
/**
@@ -19,35 +19,68 @@ class Magic {
1919
* Magic constructor.
2020
*
2121
* @param {ArrayBuffer} buf
22-
* @param {Object[]} [opPatterns]
22+
* @param {Object} prevOp
2323
*/
24-
constructor(buf, opPatterns) {
24+
constructor(buf, opPatterns, prevOp) {
2525
this.inputBuffer = new Uint8Array(buf);
2626
this.inputStr = Utils.arrayBufferToStr(buf);
27-
this.opPatterns = opPatterns || Magic._generateOpPatterns();
27+
this.opPatterns = opPatterns || Magic._generateOpCriteria();
28+
this.prevOp = prevOp;
2829
}
2930

3031
/**
31-
* Finds operations that claim to be able to decode the input based on regular
32-
* expression matches.
32+
* Finds operations that claim to be able to decode the input based on
33+
* regular expression matches.
3334
*
34-
* @returns {Object[]}
35+
* @param {[Object]} opPatterns
36+
* @returns {Array}
3537
*/
36-
findMatchingOps() {
38+
inputRegexMatch(opPatterns) {
3739
const matches = [];
3840

39-
for (let i = 0; i < this.opPatterns.length; i++) {
40-
const pattern = this.opPatterns[i],
41-
regex = new RegExp(pattern.match, pattern.flags);
41+
for (let i = 0; i < opPatterns.length; i++) {
42+
const pattern = opPatterns[i];
43+
4244

43-
if (regex.test(this.inputStr)) {
45+
if (pattern.match.test(this.inputStr)) {
4446
matches.push(pattern);
4547
}
4648
}
4749

4850
return matches;
4951
}
5052

53+
/**
54+
* Finds operations that claim to be able to decode the input based on entropy
55+
* matches.
56+
*
57+
* @param {[Object]} opPatterns
58+
* @returns {Array}
59+
*/
60+
entropyInputMatch(opPatterns) {
61+
const matches = [];
62+
63+
const entropyOfInput = this.calcEntropy();
64+
65+
for (let i = 0; i < opPatterns.length; i++) {
66+
const currOp = opPatterns[i];
67+
if ((entropyOfInput > currOp.entropy[0]) && (entropyOfInput < currOp.entropy[1]))
68+
matches.push(currOp);
69+
}
70+
return matches;
71+
}
72+
73+
/**
74+
* Finds operations that claim to be able to decode the input based on criteria.
75+
*
76+
* @returns {Object[]}
77+
*/
78+
findMatchingInputOps() {
79+
let matches = this.inputRegexMatch(this.opPatterns.regex);
80+
matches = matches.concat(this.entropyInputMatch(this.opPatterns.entropy));
81+
return [...new Set(matches)];
82+
}
83+
5184
/**
5285
* Attempts to detect the language of the input by comparing its byte frequency
5386
* to that of several known languages.
@@ -264,6 +297,35 @@ class Magic {
264297
return results;
265298
}
266299

300+
/**
301+
*
302+
*/
303+
checkRegexes(regexes) {
304+
for (const elem of regexes) {
305+
const regex = new RegExp(elem.match, elem.flags);
306+
if (regex.test(this.inputStr))
307+
return true;
308+
}
309+
return false;
310+
}
311+
/**
312+
*
313+
*/
314+
checkOutputFromPrevious() {
315+
let score = 0;
316+
if ("regex" in this.prevOp.output) {
317+
if (this.checkRegexes(this.prevOp.output.regex)) score++;
318+
}
319+
if ("entropy" in this.prevOp.output) {
320+
const inputEntropy = this.calcEntropy();
321+
if ((inputEntropy > this.prevOp.output.entropy[0]) && (inputEntropy < this.prevOp.output.entropy[1])) score++;
322+
}
323+
if ("mime" in this.prevOp.output) {
324+
if (isType(this.prevOp.output.mime, this.inputBuffer)) score++;
325+
}
326+
return score > 0;
327+
}
328+
267329
/**
268330
* Speculatively executes matching operations, recording metadata of each result.
269331
*
@@ -281,8 +343,15 @@ class Magic {
281343
if (depth < 0) return [];
282344

283345
// Find any operations that can be run on this data
284-
const matchingOps = this.findMatchingOps();
285346

347+
if (this.prevOp) {
348+
if ("output" in this.prevOp) {
349+
if (!(this.checkOutputFromPrevious())) {
350+
return [];
351+
}
352+
}
353+
}
354+
const matchingOps = this.findMatchingInputOps();
286355
let results = [];
287356

288357
// Record the properties of the current data
@@ -305,8 +374,7 @@ class Magic {
305374
const opConfig = {
306375
op: op.op,
307376
args: op.args
308-
},
309-
output = await this._runRecipe([opConfig]);
377+
}, output = await this._runRecipe([opConfig]);
310378

311379
// If the recipe is repeating and returning the same data, do not continue
312380
if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
@@ -318,7 +386,8 @@ class Magic {
318386
return;
319387
}
320388

321-
const magic = new Magic(output, this.opPatterns),
389+
390+
const magic = new Magic(output, this.opPatterns, OperationConfig[op.op]),
322391
speculativeResults = await magic.speculativeExecution(
323392
depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);
324393

@@ -330,7 +399,7 @@ class Magic {
330399
const bfEncodings = await this.bruteForce();
331400

332401
await Promise.all(bfEncodings.map(async enc => {
333-
const magic = new Magic(enc.data, this.opPatterns),
402+
const magic = new Magic(enc.data, this.opPatterns, undefined),
334403
bfResults = await magic.speculativeExecution(
335404
depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib);
336405

@@ -447,24 +516,34 @@ class Magic {
447516
* @private
448517
* @returns {Object[]}
449518
*/
450-
static _generateOpPatterns() {
451-
const opPatterns = [];
519+
static _generateOpCriteria() {
520+
const opCriteria = {
521+
regex: [],
522+
entropy: []
523+
};
452524

453525
for (const op in OperationConfig) {
454-
if (!("patterns" in OperationConfig[op])) continue;
455-
456-
OperationConfig[op].patterns.forEach(pattern => {
457-
opPatterns.push({
458-
op: op,
459-
match: pattern.match,
460-
flags: pattern.flags,
461-
args: pattern.args,
462-
useful: pattern.useful || false
463-
});
464-
});
526+
if ("input" in OperationConfig[op]) {
527+
if ("regex" in OperationConfig[op].input)
528+
OperationConfig[op].input.regex.forEach(pattern => {
529+
opCriteria.regex.push({
530+
op: op,
531+
match: new RegExp(pattern.match, pattern.flags),
532+
args: pattern.args,
533+
useful: pattern.useful || false
534+
});
535+
});
536+
if ("entropy" in OperationConfig[op].input) {
537+
opCriteria.entropy.push({
538+
op: op,
539+
entropy: OperationConfig[op].input.entropy.input,
540+
args: OperationConfig[op].input.entropy.args
541+
});
542+
}
543+
}
465544
}
466545

467-
return opPatterns;
546+
return opCriteria;
468547
}
469548

470549
/**

‎src/core/lib/MagicCriteria.mjs

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/**
2+
* Constants for the entropy of text.
3+
*
4+
* @author n1073645 [[email protected]]
5+
* @copyright Crown Copyright 2020
6+
* @license Apache-2.0
7+
*/
8+
export const compressedToDecompressed = [6.5, 8];
9+
10+
export const binary = [1, 1.5];
11+
12+
export const entropyOfText = [3.5, 6];

‎src/core/operations/A1Z26CipherDecode.mjs

+36
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,42 @@ class A1Z26CipherDecode extends Operation {
3333
value: DELIM_OPTIONS
3434
}
3535
];
36+
this.checks = {
37+
input: {
38+
regex: [
39+
{
40+
match: "^\\s*([12]?[0-9] )+[12]?[0-9]\\s*$",
41+
flags: "",
42+
args: ["Space"]
43+
},
44+
{
45+
match: "^\\s*([12]?[0-9],)+[12]?[0-9]\\s*$",
46+
flags: "",
47+
args: ["Comma"]
48+
},
49+
{
50+
match: "^\\s*([12]?[0-9];)+[12]?[0-9]\\s*$",
51+
flags: "",
52+
args: ["Semi-colon"]
53+
},
54+
{
55+
match: "^\\s*([12]?[0-9]:)+[12]?[0-9]\\s*$",
56+
flags: "",
57+
args: ["Colon"]
58+
},
59+
{
60+
match: "^\\s*([12]?[0-9]\\n)+[12]?[0-9]\\s*$",
61+
flags: "",
62+
args: ["Line feed"]
63+
},
64+
{
65+
match: "^\\s*([12]?[0-9]\\r\\n)+[12]?[0-9]\\s*$",
66+
flags: "",
67+
args: ["CRLF"]
68+
}
69+
]
70+
}
71+
};
3672
}
3773

3874
/**

0 commit comments

Comments
 (0)