335 lines
9.9 KiB
JavaScript
335 lines
9.9 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { createHash } from "node:crypto";
|
|
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { inflateRawSync } from "node:zlib";
|
|
|
|
/**
|
|
* @file scripts/sync-korean-stocks.mjs
|
|
* @description KIS 종목 마스터 파일(KOSPI/KOSDAQ)로 검색 인덱스 JSON을 자동 갱신합니다.
|
|
*/
|
|
|
|
const OUTPUT_FILE_PATH = path.resolve(
|
|
process.cwd(),
|
|
"features/trade/data/korean-stocks.json",
|
|
);
|
|
|
|
const SOURCE_CONFIGS = [
|
|
{
|
|
market: "KOSPI",
|
|
tailWidth: 228,
|
|
url: "https://new.real.download.dws.co.kr/common/master/kospi_code.mst.zip",
|
|
},
|
|
{
|
|
market: "KOSDAQ",
|
|
tailWidth: 222,
|
|
url: "https://new.real.download.dws.co.kr/common/master/kosdaq_code.mst.zip",
|
|
},
|
|
];
|
|
|
|
const MIN_EXPECTED_TOTAL = 3000;
|
|
const MIN_EXPECTED_PER_MARKET = 1000;
|
|
|
|
/**
|
|
* CLI 진입점
|
|
* @see scripts/sync-korean-stocks.mjs main() 종목 인덱스 갱신 파이프라인을 실행합니다.
|
|
*/
|
|
async function main() {
|
|
const options = parseCliArgs(process.argv.slice(2));
|
|
|
|
if (options.help) {
|
|
printHelp();
|
|
return;
|
|
}
|
|
|
|
const allItems = [];
|
|
for (const source of SOURCE_CONFIGS) {
|
|
const zipBuffer = await downloadBinary(source.url);
|
|
const { fileName, data } = extractFirstZipEntry(zipBuffer);
|
|
const parsed = parseMasterRows(data, source.market, source.tailWidth);
|
|
console.log(
|
|
`[sync:stocks] ${source.market} parsed ${parsed.length} rows from ${fileName}`,
|
|
);
|
|
allItems.push(...parsed);
|
|
}
|
|
|
|
const normalized = normalizeItems(allItems);
|
|
validateCounts(normalized);
|
|
|
|
const nextJson = `${JSON.stringify(normalized, null, 2)}\n`;
|
|
const nextHash = sha256(nextJson);
|
|
|
|
if (options.dryRun) {
|
|
console.log(`[sync:stocks] dry-run complete, rows=${normalized.length}, hash=${nextHash}`);
|
|
return;
|
|
}
|
|
|
|
if (options.check) {
|
|
const currentJson = await readFile(OUTPUT_FILE_PATH, "utf8").catch(() => "");
|
|
const currentHash = sha256(currentJson);
|
|
if (currentJson !== nextJson) {
|
|
console.error(
|
|
`[sync:stocks] out-of-date: current=${currentHash}, next=${nextHash}`,
|
|
);
|
|
console.error("[sync:stocks] run `npm run sync:stocks` to update.");
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
|
|
console.log(`[sync:stocks] up-to-date, rows=${normalized.length}, hash=${currentHash}`);
|
|
return;
|
|
}
|
|
|
|
await writeFileAtomically(OUTPUT_FILE_PATH, nextJson);
|
|
console.log(`[sync:stocks] updated ${OUTPUT_FILE_PATH}`);
|
|
console.log(`[sync:stocks] rows=${normalized.length}, hash=${nextHash}`);
|
|
}
|
|
|
|
/**
|
|
* CLI 인자 파서
|
|
* @see scripts/sync-korean-stocks.mjs main() 실행 모드를 결정합니다.
|
|
*/
|
|
function parseCliArgs(args) {
|
|
return {
|
|
check: args.includes("--check"),
|
|
dryRun: args.includes("--dry-run"),
|
|
help: args.includes("--help") || args.includes("-h"),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* 도움말 출력
|
|
* @see scripts/sync-korean-stocks.mjs parseCliArgs() 전달된 옵션을 안내합니다.
|
|
*/
|
|
function printHelp() {
|
|
console.log("Usage: node scripts/sync-korean-stocks.mjs [--check] [--dry-run]");
|
|
console.log("");
|
|
console.log("Options:");
|
|
console.log(" --check compare generated JSON with current file and exit 1 on diff");
|
|
console.log(" --dry-run parse and validate data without writing output");
|
|
console.log(" -h, --help show this help message");
|
|
}
|
|
|
|
/**
|
|
* 원격 바이너리 다운로드
|
|
* @see scripts/sync-korean-stocks.mjs main() KIS 마스터 ZIP 파일을 가져옵니다.
|
|
*/
|
|
async function downloadBinary(url) {
|
|
const response = await fetch(url, {
|
|
headers: {
|
|
"user-agent": "auto-trade-stock-sync/1.0",
|
|
},
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`Failed to download ${url} (${response.status} ${response.statusText})`);
|
|
}
|
|
|
|
const arrayBuffer = await response.arrayBuffer();
|
|
return Buffer.from(arrayBuffer);
|
|
}
|
|
|
|
/**
|
|
* ZIP 첫 번째 엔트리 추출
|
|
* @see scripts/sync-korean-stocks.mjs main() ZIP에서 .mst 본문을 읽어옵니다.
|
|
*/
|
|
function extractFirstZipEntry(zipBuffer) {
|
|
const eocdOffset = findEndOfCentralDirectory(zipBuffer);
|
|
const totalEntries = zipBuffer.readUInt16LE(eocdOffset + 10);
|
|
const centralDirectoryOffset = zipBuffer.readUInt32LE(eocdOffset + 16);
|
|
|
|
if (totalEntries < 1) {
|
|
throw new Error("ZIP has no entries.");
|
|
}
|
|
|
|
const cdSignature = zipBuffer.readUInt32LE(centralDirectoryOffset);
|
|
if (cdSignature !== 0x02014b50) {
|
|
throw new Error("Invalid central directory signature.");
|
|
}
|
|
|
|
const method = zipBuffer.readUInt16LE(centralDirectoryOffset + 10);
|
|
const compressedSize = zipBuffer.readUInt32LE(centralDirectoryOffset + 20);
|
|
const uncompressedSize = zipBuffer.readUInt32LE(centralDirectoryOffset + 24);
|
|
const fileNameLength = zipBuffer.readUInt16LE(centralDirectoryOffset + 28);
|
|
const extraLength = zipBuffer.readUInt16LE(centralDirectoryOffset + 30);
|
|
const commentLength = zipBuffer.readUInt16LE(centralDirectoryOffset + 32);
|
|
const localHeaderOffset = zipBuffer.readUInt32LE(centralDirectoryOffset + 42);
|
|
const fileNameStart = centralDirectoryOffset + 46;
|
|
const fileNameEnd = fileNameStart + fileNameLength;
|
|
const fileName = zipBuffer.subarray(fileNameStart, fileNameEnd).toString("utf8");
|
|
|
|
const _unused = extraLength + commentLength;
|
|
void _unused;
|
|
|
|
const localSignature = zipBuffer.readUInt32LE(localHeaderOffset);
|
|
if (localSignature !== 0x04034b50) {
|
|
throw new Error("Invalid local header signature.");
|
|
}
|
|
|
|
const localNameLength = zipBuffer.readUInt16LE(localHeaderOffset + 26);
|
|
const localExtraLength = zipBuffer.readUInt16LE(localHeaderOffset + 28);
|
|
const dataStart = localHeaderOffset + 30 + localNameLength + localExtraLength;
|
|
const dataEnd = dataStart + compressedSize;
|
|
const compressedData = zipBuffer.subarray(dataStart, dataEnd);
|
|
|
|
let data;
|
|
if (method === 0) {
|
|
data = compressedData;
|
|
} else if (method === 8) {
|
|
data = inflateRawSync(compressedData);
|
|
} else {
|
|
throw new Error(`Unsupported ZIP compression method: ${method}`);
|
|
}
|
|
|
|
if (uncompressedSize !== 0 && data.length !== uncompressedSize) {
|
|
throw new Error(
|
|
`Uncompressed size mismatch for ${fileName}: expected=${uncompressedSize}, actual=${data.length}`,
|
|
);
|
|
}
|
|
|
|
return { fileName, data };
|
|
}
|
|
|
|
/**
|
|
* EOCD(End Of Central Directory) 오프셋 탐색
|
|
* @see scripts/sync-korean-stocks.mjs extractFirstZipEntry() ZIP 중앙 디렉터리 위치를 찾습니다.
|
|
*/
|
|
function findEndOfCentralDirectory(zipBuffer) {
|
|
const minOffset = Math.max(0, zipBuffer.length - 65557);
|
|
for (let i = zipBuffer.length - 22; i >= minOffset; i -= 1) {
|
|
if (zipBuffer.readUInt32LE(i) === 0x06054b50) {
|
|
return i;
|
|
}
|
|
}
|
|
throw new Error("EOCD signature not found in ZIP.");
|
|
}
|
|
|
|
/**
|
|
* .mst 텍스트 파싱
|
|
* @see scripts/sync-korean-stocks.mjs main() symbol/name/standardCode를 추출합니다.
|
|
*/
|
|
function parseMasterRows(mstBuffer, market, tailWidth) {
|
|
const decoder = new TextDecoder("euc-kr");
|
|
const text = decoder.decode(mstBuffer);
|
|
const lines = text.split(/\r?\n/);
|
|
const items = [];
|
|
|
|
for (const rawLine of lines) {
|
|
const line = rawLine.trimEnd();
|
|
if (!line) continue;
|
|
if (line.length <= tailWidth + 21) continue;
|
|
|
|
const part1 = line.slice(0, line.length - tailWidth);
|
|
const symbol = part1.slice(0, 9).trim();
|
|
const standardCode = part1.slice(9, 21).trim();
|
|
const name = part1.slice(21).trim();
|
|
|
|
if (!/^\d{6}$/.test(symbol)) continue;
|
|
if (!standardCode || !name) continue;
|
|
|
|
items.push({
|
|
symbol,
|
|
name,
|
|
market,
|
|
standardCode,
|
|
});
|
|
}
|
|
|
|
return items;
|
|
}
|
|
|
|
/**
|
|
* 아이템 정규화/중복 제거/정렬
|
|
* @see scripts/sync-korean-stocks.mjs main() 검색 인덱스 최종 포맷을 만듭니다.
|
|
*/
|
|
function normalizeItems(items) {
|
|
const uniqueBySymbol = new Map();
|
|
|
|
for (const item of items) {
|
|
const existing = uniqueBySymbol.get(item.symbol);
|
|
if (!existing) {
|
|
uniqueBySymbol.set(item.symbol, item);
|
|
continue;
|
|
}
|
|
|
|
const same =
|
|
existing.market === item.market &&
|
|
existing.name === item.name &&
|
|
existing.standardCode === item.standardCode;
|
|
|
|
if (!same) {
|
|
throw new Error(
|
|
`Duplicate symbol conflict (${item.symbol}): ${JSON.stringify(existing)} <> ${JSON.stringify(item)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
return [...uniqueBySymbol.values()].sort((a, b) => {
|
|
const bySymbol = a.symbol.localeCompare(b.symbol);
|
|
if (bySymbol !== 0) return bySymbol;
|
|
return a.market.localeCompare(b.market);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* 기본 품질 검증
|
|
* @see scripts/sync-korean-stocks.mjs main() 비정상적으로 적은 데이터면 실패 처리합니다.
|
|
*/
|
|
function validateCounts(items) {
|
|
if (items.length < MIN_EXPECTED_TOTAL) {
|
|
throw new Error(
|
|
`Total row count is too small: ${items.length} < ${MIN_EXPECTED_TOTAL}`,
|
|
);
|
|
}
|
|
|
|
const marketCount = items.reduce(
|
|
(acc, item) => {
|
|
acc[item.market] += 1;
|
|
return acc;
|
|
},
|
|
{ KOSPI: 0, KOSDAQ: 0 },
|
|
);
|
|
|
|
if (marketCount.KOSPI < MIN_EXPECTED_PER_MARKET) {
|
|
throw new Error(
|
|
`KOSPI row count is too small: ${marketCount.KOSPI} < ${MIN_EXPECTED_PER_MARKET}`,
|
|
);
|
|
}
|
|
|
|
if (marketCount.KOSDAQ < MIN_EXPECTED_PER_MARKET) {
|
|
throw new Error(
|
|
`KOSDAQ row count is too small: ${marketCount.KOSDAQ} < ${MIN_EXPECTED_PER_MARKET}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 원자적 파일 저장
|
|
* @see scripts/sync-korean-stocks.mjs main() 저장 도중 파일 손상을 방지합니다.
|
|
*/
|
|
async function writeFileAtomically(targetPath, content) {
|
|
const dir = path.dirname(targetPath);
|
|
const tempPath = path.join(
|
|
dir,
|
|
`.${path.basename(targetPath)}.${process.pid}.${Date.now()}.tmp`,
|
|
);
|
|
|
|
await mkdir(dir, { recursive: true });
|
|
await writeFile(tempPath, content, "utf8");
|
|
await rename(tempPath, targetPath);
|
|
}
|
|
|
|
/**
|
|
* SHA-256 해시
|
|
* @see scripts/sync-korean-stocks.mjs main() 변경 여부를 간단히 비교합니다.
|
|
*/
|
|
function sha256(value) {
|
|
return createHash("sha256").update(value).digest("hex");
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(`[sync:stocks] ${error instanceof Error ? error.message : String(error)}`);
|
|
process.exit(1);
|
|
});
|