stacker.news/worker/trust.js

205 lines
7.8 KiB
JavaScript
Raw Normal View History

Convert worker to ESM (#500) * Convert worker to ESM To use ESM for the worker, I created a package.json file in worker/ with `{ type: "module" }` as its sole content. I then rewrote every import to use ESM syntax. I also tried to set `{ type: "module" }` in the root package.json file to also use ESM in next.config.js. However, this resulted in a weird problem: default imports were now getting imported as objects in this shape: `{ default: <defaultImport> }`. Afaik, this should only be the case if you use "import * as foo from 'bar'" syntax: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#default_import This is fixed by not using `{ type: "module" }` for some reason. However, then, next.config.js also doesn't support ESM import syntax anymore. The documentation says that if you want to use ESM, you can use next.config.mjs: https://nextjs.org/docs/pages/api-reference/next-config-js But I didn't want to use MJS extension since I don't have any experience with it. For example, not sure if it's good style to mix JS with MJS etc. So I kept the CJS import syntax there. * Ignore worker/ during linting I wasn't able to fix the following errors: /home/runner/work/stacker.news/stacker.news/worker/auction.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/auction.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/earn.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/earn.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/index.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/index.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/nostr.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/nostr.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/ots.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/ots.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/repin.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/repin.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/search.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/search.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/streak.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/streak.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/trust.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/trust.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/views.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/views.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/wallet.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/wallet.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) I tried to tell babel where to find the babel config file (.babelrc), specifying the babel config in worker/package.json under "babel", using babel.config.json etc. to no avail. However, afaict, we don't need babel for the worker since it won't run in a browser. Babel is only used to transpile code to target browsers. But it still would be nice to lint the worker code with standard. But we can figure this out later. * Fix worker imports from lib/ and api/ This fixes the issue that we can't use `{ "type": "module" }` in the root package.json since it breaks the app with this error: app | TypeError: next_auth_providers_credentials__WEBPACK_IMPORTED_MODULE_2__ is not a function app | at eval (webpack-internal:///./pages/api/auth/[...nextauth].js:218:20) app | at process.processTicksAndRejections (node:internal/process/task_queues:95:5) app | LND GRPC connection successful app | - error pages/api/auth/[...nextauth].js (139:2) @ CredentialsProvider app | - error Error [TypeError]: next_auth_providers_credentials__WEBPACK_IMPORTED_MODULE_2__ is not a function app | at eval (webpack-internal:///./pages/api/auth/[...nextauth].js:218:20) { app | digest: undefined app | } app | 137 | app | 138 | const providers = [ app | > 139 | CredentialsProvider({ app | | ^ app | 140 | id: 'lightning', app | 141 | name: 'Lightning', app | 142 | credentials: { app | TypeError: next_auth_providers_credentials__WEBPACK_IMPORTED_MODULE_2__ is not a function app | at eval (webpack-internal:///./pages/api/auth/[...nextauth].js:218:20) app | at process.processTicksAndRejections (node:internal/process/task_queues:95:5) build but we need to tell the worker that the files are MJS, else we get this error: worker | file:///app/worker/wallet.js:3 worker | import { datePivot } from '../lib/time.js' worker | ^^^^^^^^^ worker | SyntaxError: Named export 'datePivot' not found. The requested module '../lib/time.js' is a CommonJS module, which may not support all module.exports as named exports. worker | CommonJS modules can always be imported via the default export, for example using: worker | worker | import pkg from '../lib/time.js'; worker | const { datePivot } = pkg; worker | worker | at ModuleJob._instantiate (node:internal/modules/esm/module_job:124:21) worker | at async ModuleJob.run (node:internal/modules/esm/module_job:190:5) worker | worker | Node.js v18.17.0 worker | worker exited with code 1 * Fix global not defined in browser context * Also ignore api/ and lib/ during linting I did not want to do this but I was not able to fix this error in any other way I tried: /home/ekzyis/programming/stacker.news/api/lnd/index.js:0:0: Parsing error: No Babel config file detected for /home/ekzyis/programming/stacker.news/api/lnd/index.js. Either disable config file checking with requ ireConfigFile: false, or configure Babel so that it can find the config files. (null) Did not want to look deeper into all this standard, eslint, babel configuration stuff ... --------- Co-authored-by: ekzyis <ek@stacker.news> Co-authored-by: Keyan <34140557+huumn@users.noreply.github.com>
2023-09-24 01:19:35 +00:00
import * as math from 'mathjs'
import { USER_ID, SN_USER_IDS } from '@/lib/constants.js'
2023-07-06 01:30:50 +00:00
2023-11-21 23:32:22 +00:00
export async function trust ({ boss, models }) {
try {
console.time('trust')
console.timeLog('trust', 'getting graph')
const graph = await getGraph(models)
console.timeLog('trust', 'computing trust')
const [vGlobal, mPersonal] = await trustGivenGraph(graph)
console.timeLog('trust', 'storing trust')
await storeTrust(models, graph, vGlobal, mPersonal)
} finally {
console.timeEnd('trust')
2022-01-17 21:47:26 +00:00
}
}
2023-07-06 01:30:50 +00:00
const MAX_DEPTH = 10
const MAX_TRUST = 1
const MIN_SUCCESS = 1
// increasing disgree_mult increases distrust when there's disagreement
// ... this cancels DISAGREE_MULT number of "successes" for every disagreement
2023-09-19 00:42:39 +00:00
const DISAGREE_MULT = 10
2023-07-06 01:30:50 +00:00
// https://en.wikipedia.org/wiki/Normal_distribution#Quantile_function
const Z_CONFIDENCE = 6.109410204869 // 99.9999999% confidence
2023-09-14 15:46:59 +00:00
const GLOBAL_ROOT = 616
2023-07-06 01:30:50 +00:00
const SEED_WEIGHT = 0.25
const AGAINST_MSAT_MIN = 1000
const MSAT_MIN = 1000
2023-09-14 15:46:59 +00:00
const SIG_DIFF = 0.1 // need to differ by at least 10 percent
2022-01-17 21:47:26 +00:00
/*
Given a graph and start this function returns an object where
the keys are the node id and their value is the trust of that node
*/
2023-07-06 01:30:50 +00:00
function trustGivenGraph (graph) {
// empty matrix of proper size nstackers x nstackers
2023-09-14 15:46:59 +00:00
let mat = math.zeros(graph.length, graph.length, 'sparse')
2023-07-06 01:30:50 +00:00
// create a map of user id to position in matrix
const posByUserId = {}
for (const [idx, val] of graph.entries()) {
posByUserId[val.id] = idx
}
2022-01-17 21:47:26 +00:00
2023-07-06 01:30:50 +00:00
// iterate over graph, inserting edges into matrix
for (const [idx, val] of graph.entries()) {
for (const { node, trust } of val.hops) {
try {
mat.set([idx, posByUserId[node]], Number(trust))
} catch (e) {
console.log('error:', idx, node, posByUserId[node], trust)
throw e
}
2022-01-17 21:47:26 +00:00
}
2023-07-06 01:30:50 +00:00
}
2022-01-17 21:47:26 +00:00
2023-07-06 01:30:50 +00:00
// perform random walk over trust matrix
// the resulting matrix columns represent the trust a user (col) has for each other user (rows)
// XXX this scales N^3 and mathjs is slow
let matT = math.transpose(mat)
const original = matT.clone()
for (let i = 0; i < MAX_DEPTH; i++) {
console.timeLog('trust', `matrix multiply ${i}`)
matT = math.multiply(original, matT)
matT = math.add(math.multiply(1 - SEED_WEIGHT, matT), math.multiply(SEED_WEIGHT, original))
}
2022-01-17 21:47:26 +00:00
2023-09-14 15:46:59 +00:00
console.timeLog('trust', 'transforming result')
2023-12-04 15:35:54 +00:00
const seedIdxs = SN_USER_IDS.map(id => posByUserId[id])
2023-09-14 15:46:59 +00:00
const isOutlier = (fromIdx, idx) => [...seedIdxs, fromIdx].includes(idx)
const sqapply = (mat, fn) => {
let idx = 0
return math.squeeze(math.apply(mat, 1, d => {
const filtered = math.filter(d, (val, fidx) => {
return val !== 0 && !isOutlier(idx, fidx[0])
})
idx++
if (filtered.length === 0) return 0
return fn(filtered)
}))
2022-01-17 21:47:26 +00:00
}
2023-09-14 15:46:59 +00:00
console.timeLog('trust', 'normalizing')
console.timeLog('trust', 'stats')
mat = math.transpose(matT)
const std = sqapply(mat, math.std) // math.squeeze(math.std(mat, 1))
const mean = sqapply(mat, math.mean) // math.squeeze(math.mean(mat, 1))
const zscore = math.map(mat, (val, idx) => {
const zstd = math.subset(std, math.index(idx[0]))
const zmean = math.subset(mean, math.index(idx[0]))
return zstd ? (val - zmean) / zstd : 0
2023-07-06 01:30:50 +00:00
})
2023-09-14 15:46:59 +00:00
console.timeLog('trust', 'minmax')
const min = sqapply(zscore, math.min) // math.squeeze(math.min(zscore, 1))
const max = sqapply(zscore, math.max) // math.squeeze(math.max(zscore, 1))
const mPersonal = math.map(zscore, (val, idx) => {
const zmin = math.subset(min, math.index(idx[0]))
const zmax = math.subset(max, math.index(idx[0]))
const zrange = zmax - zmin
if (val > zmax) return MAX_TRUST
return zrange ? (val - zmin) / zrange : 0
})
const vGlobal = math.squeeze(math.row(mPersonal, posByUserId[GLOBAL_ROOT]))
2023-07-06 01:30:50 +00:00
2023-09-14 15:46:59 +00:00
return [vGlobal, mPersonal]
2022-01-17 21:47:26 +00:00
}
/*
2023-07-06 01:30:50 +00:00
graph is returned as json in adjacency list where edges are the trust value 0-1
graph = [
{ id: node1, hops: [{node : node2, trust: trust12}, {node: node3, trust: trust13}] },
...
]
2022-01-17 21:47:26 +00:00
*/
async function getGraph (models) {
2023-07-06 01:30:50 +00:00
return await models.$queryRaw`
2023-09-14 15:46:59 +00:00
SELECT id, json_agg(json_build_object(
2023-07-06 01:30:50 +00:00
'node', oid,
'trust', CASE WHEN total_trust > 0 THEN trust / total_trust::float ELSE 0 END)) AS hops
FROM (
WITH user_votes AS (
SELECT "ItemAct"."userId" AS user_id, users.name AS name, "ItemAct"."itemId" AS item_id, min("ItemAct".created_at) AS act_at,
users.created_at AS user_at, "ItemAct".act = 'DONT_LIKE_THIS' AS against,
count(*) OVER (partition by "ItemAct"."userId") AS user_vote_count
FROM "ItemAct"
JOIN "Item" ON "Item".id = "ItemAct"."itemId" AND "ItemAct".act IN ('FEE', 'TIP', 'DONT_LIKE_THIS')
AND "Item"."parentId" IS NULL AND NOT "Item".bio AND "Item"."userId" <> "ItemAct"."userId"
JOIN users ON "ItemAct"."userId" = users.id AND users.id <> ${USER_ID.anon}
backend payment optimism (#1195) * wip backend optimism * another inch * make action state transitions only happen once * another inch * almost ready for testing * use interactive txs * another inch * ready for basic testing * lint fix * inches * wip item update * get item update to work * donate and downzap * inchy inch * fix territory paid actions * wip usePaidMutation * usePaidMutation error handling * PENDING_HELD and HELD transitions, gql paidAction return types * mostly working pessimism * make sure invoice field is present in optimisticResponse * inches * show optimistic values to current me * first pass at notifications and payment status reporting * fix migration to have withdrawal hash * reverse optimism on payment failure * Revert "Optimistic updates via pending sats in item context (#1229)" This reverts commit 93713b33df9bc3701dc5a692b86a04ff64e8cfb1. * add onCompleted to usePaidMutation * onPaid and onPayError for new comments * use 'IS DISTINCT FROM' for NULL invoiceActionState columns * make usePaidMutation easier to read * enhance invoice qr * prevent actions on unpaid items * allow navigation to action's invoice * retry create item * start edit window after item is paid for * fix ux of retries from notifications * refine retries * fix optimistic downzaps * remember item updates can't be retried * store reference to action item in invoice * remove invoice modal layout shift * fix destructuring * fix zap undos * make sure ItemAct is paid in aggregate queries * dont toast on long press zap undo * fix delete and remindme bots * optimistic poll votes with retries * fix retry notifications and invoice item context * fix pessimisitic typo * item mentions and mention notifications * dont show payment retry on item popover * make bios work * refactor paidAction transitions * remove stray console.log * restore docker compose nwc settings * add new todos * persist qr modal on post submission + unify item form submission * fix post edit threshold * make bounty payments work * make job posting work * remove more store procedure usage ... document serialization concerns * dont use dynamic imports for paid action modules * inline comment denormalization * create item starts with median votes * fix potential of serialization anomalies in zaps * dont trigger notification indicator on successful paid action invoices * ignore invoiceId on territory actions and add optimistic concurrency control * begin docs for paid actions * better error toasts and fix apollo cache warnings * small documentation enhancements * improve paid action docs * optimistic concurrency control for territory updates * use satsToMsats and msatsToSats helpers * explictly type raw query template parameters * improve consistency of nested relation names * complete paid action docs * useEffect for canEdit on payment * make sure invoiceId is provided when required * don't return null when expecting array * remove buy credits * move verifyPayment to paidAction * fix comments invoicePaidAt time zone * close nwc connections once * grouped logs for paid actions * stop invoiceWaitUntilPaid if not attempting to pay * allow actionState to transition directly from HELD to PAID * make paid mutation wait until pessimistic are fully paid * change button text when form submits/pays * pulsing form submit button * ignore me in notification indicator for territory subscription * filter unpaid items from more queries * fix donation stike timing * fix pending poll vote * fix recent item notifcation padding * no default form submitting button text * don't show paying on submit button on free edits * fix territory autorenew with fee credits * reorg readme * allow jobs to be editted forever * fix image uploads * more filter fixes for aggregate views * finalize paid action invoice expirations * remove unnecessary async * keep clientside cache normal/consistent * add more detail to paid action doc * improve paid action table * remove actionType guard * fix top territories * typo api/paidAction/README.md Co-authored-by: ekzyis <ek@stacker.news> * typo components/use-paid-mutation.js Co-authored-by: ekzyis <ek@stacker.news> * Apply suggestions from code review Co-authored-by: ekzyis <ek@stacker.news> * encorporate ek feeback * more ek suggestions * fix 'cost to post' hover on items * Apply suggestions from code review Co-authored-by: ekzyis <ek@stacker.news> --------- Co-authored-by: ekzyis <ek@stacker.news>
2024-07-01 17:02:29 +00:00
WHERE "ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID'
2023-07-06 01:30:50 +00:00
GROUP BY user_id, name, item_id, user_at, against
HAVING CASE WHEN
"ItemAct".act = 'DONT_LIKE_THIS' THEN sum("ItemAct".msats) > ${AGAINST_MSAT_MIN}
ELSE sum("ItemAct".msats) > ${MSAT_MIN} END
),
user_pair AS (
SELECT a.user_id AS a_id, b.user_id AS b_id,
count(*) FILTER(WHERE a.act_at > b.act_at AND a.against = b.against) AS before,
count(*) FILTER(WHERE b.act_at > a.act_at AND a.against = b.against) AS after,
count(*) FILTER(WHERE a.against <> b.against) * ${DISAGREE_MULT} AS disagree,
b.user_vote_count AS b_total, a.user_vote_count AS a_total
FROM user_votes a
JOIN user_votes b ON a.item_id = b.item_id
WHERE a.user_id <> b.user_id
GROUP BY a.user_id, a.user_vote_count, b.user_id, b.user_vote_count
),
trust_pairs AS (
SELECT a_id AS id, b_id AS oid,
CASE WHEN before - disagree >= ${MIN_SUCCESS} AND b_total - after > 0 THEN
confidence(before - disagree, b_total - after, ${Z_CONFIDENCE})
ELSE 0 END AS trust
FROM user_pair
WHERE NOT (b_id = ANY (${SN_USER_IDS}))
2023-07-06 01:30:50 +00:00
UNION ALL
2023-09-14 15:46:59 +00:00
SELECT a_id AS id, seed_id AS oid, ${MAX_TRUST}::numeric as trust
2023-12-04 15:35:54 +00:00
FROM user_pair, unnest(${SN_USER_IDS}::int[]) seed_id
2023-07-06 01:30:50 +00:00
GROUP BY a_id, a_total, seed_id
2023-09-14 15:46:59 +00:00
UNION ALL
SELECT a_id AS id, a_id AS oid, ${MAX_TRUST}::float as trust
FROM user_pair
2023-07-06 01:30:50 +00:00
)
SELECT id, oid, trust, sum(trust) OVER (PARTITION BY id) AS total_trust
FROM trust_pairs
) a
GROUP BY a.id
ORDER BY id ASC`
2022-01-17 21:47:26 +00:00
}
2023-09-14 15:46:59 +00:00
async function storeTrust (models, graph, vGlobal, mPersonal) {
2022-01-17 21:47:26 +00:00
// convert nodeTrust into table literal string
2023-09-14 15:46:59 +00:00
let globalValues = ''
let personalValues = ''
vGlobal.forEach((val, [idx]) => {
if (isNaN(val)) return
if (globalValues) globalValues += ','
globalValues += `(${graph[idx].id}, ${val}::FLOAT)`
if (personalValues) personalValues += ','
personalValues += `(${GLOBAL_ROOT}, ${graph[idx].id}, ${val}::FLOAT)`
})
math.forEach(mPersonal, (val, [fromIdx, toIdx]) => {
const globalVal = vGlobal.get([toIdx])
if (isNaN(val) || val - globalVal <= SIG_DIFF) return
if (personalValues) personalValues += ','
personalValues += `(${graph[fromIdx].id}, ${graph[toIdx].id}, ${val}::FLOAT)`
})
2022-01-17 21:47:26 +00:00
// update the trust of each user in graph
await models.$transaction([
models.$executeRaw`UPDATE users SET trust = 0`,
2023-07-26 16:01:31 +00:00
models.$executeRawUnsafe(
`UPDATE users
SET trust = g.trust
2023-09-14 15:46:59 +00:00
FROM (values ${globalValues}) g(id, trust)
WHERE users.id = g.id`),
models.$executeRawUnsafe(
`INSERT INTO "Arc" ("fromId", "toId", "zapTrust")
SELECT id, oid, trust
FROM (values ${personalValues}) g(id, oid, trust)
ON CONFLICT ("fromId", "toId") DO UPDATE SET "zapTrust" = EXCLUDED."zapTrust"`
)
])
2022-01-17 21:47:26 +00:00
}