78 lines
2.5 KiB
JavaScript
78 lines
2.5 KiB
JavaScript
import { createReadStream, createWriteStream, unlinkSync } from 'fs'
|
|
import unzipper from 'unzipper'
|
|
import csvParser from 'csv-parser'
|
|
import stream from 'stream'
|
|
import { SANCTIONED_COUNTRY_CODES } from '../lib/constants.js'
|
|
|
|
const IPV4_URL = 'https://ipapi.is/data/geolocationDatabaseIPv4.csv.zip'
|
|
const IPV6_URL = 'https://ipapi.is/data/geolocationDatabaseIPv6.csv.zip'
|
|
|
|
export async function ofac ({ models }) {
|
|
const ipv4CSVPath = 'ipv4.temp.csv'
|
|
const ipv6CSVPath = 'ipv6.temp.csv'
|
|
|
|
async function loadCSVIntoDatabase (csvFilePath) {
|
|
const results = []
|
|
|
|
return new Promise((resolve, reject) => {
|
|
createReadStream(csvFilePath)
|
|
.pipe(csvParser())
|
|
.on('data', (data) => {
|
|
if (SANCTIONED_COUNTRY_CODES.includes(data.country_code)) {
|
|
results.push({
|
|
startIP: data.start_ip,
|
|
endIP: data.end_ip,
|
|
country: data.country,
|
|
countryCode: data.country_code
|
|
})
|
|
}
|
|
})
|
|
.on('end', async () => {
|
|
console.log('results', results.length)
|
|
await models.$queryRaw`
|
|
INSERT INTO "OFAC" ("startIP", "endIP", "country", "countryCode")
|
|
SELECT "startIP", "endIP", "country", "countryCode" FROM json_populate_recordset(null::"OFAC", ${JSON.stringify(results)}::JSON)`
|
|
console.log('Data imported into the database')
|
|
resolve()
|
|
})
|
|
.on('error', reject)
|
|
})
|
|
}
|
|
|
|
try {
|
|
await downloadAndUnzipCSV(IPV4_URL, ipv4CSVPath)
|
|
await downloadAndUnzipCSV(IPV6_URL, ipv6CSVPath)
|
|
await models.$executeRaw`TRUNCATE TABLE "OFAC"`
|
|
await loadCSVIntoDatabase(ipv4CSVPath)
|
|
await loadCSVIntoDatabase(ipv6CSVPath)
|
|
} finally {
|
|
unlinkSync(ipv4CSVPath)
|
|
unlinkSync(ipv6CSVPath)
|
|
}
|
|
}
|
|
|
|
async function downloadAndUnzipCSV (url, outputFilePath) {
|
|
const response = await fetch(url)
|
|
if (!response.ok) throw new Error(`Failed to fetch: ${response.statusText}`)
|
|
|
|
return new Promise((resolve, reject) => {
|
|
stream.Readable.fromWeb(response.body)
|
|
.pipe(unzipper.Parse())
|
|
.on('entry', function (entry) {
|
|
console.log('Extracting', entry.path)
|
|
if (entry.path.endsWith('.csv')) {
|
|
const fileStream = createWriteStream(outputFilePath)
|
|
entry.pipe(fileStream)
|
|
fileStream.on('finish', () => {
|
|
console.log('File write completed:', outputFilePath)
|
|
resolve()
|
|
})
|
|
fileStream.on('error', reject)
|
|
} else {
|
|
entry.autodrain()
|
|
}
|
|
})
|
|
.on('error', reject)
|
|
})
|
|
}
|