stacker.news/worker/earn.js

247 lines
11 KiB
JavaScript
Raw Normal View History

Convert worker to ESM (#500) * Convert worker to ESM To use ESM for the worker, I created a package.json file in worker/ with `{ type: "module" }` as its sole content. I then rewrote every import to use ESM syntax. I also tried to set `{ type: "module" }` in the root package.json file to also use ESM in next.config.js. However, this resulted in a weird problem: default imports were now getting imported as objects in this shape: `{ default: <defaultImport> }`. Afaik, this should only be the case if you use "import * as foo from 'bar'" syntax: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#default_import This is fixed by not using `{ type: "module" }` for some reason. However, then, next.config.js also doesn't support ESM import syntax anymore. The documentation says that if you want to use ESM, you can use next.config.mjs: https://nextjs.org/docs/pages/api-reference/next-config-js But I didn't want to use MJS extension since I don't have any experience with it. For example, not sure if it's good style to mix JS with MJS etc. So I kept the CJS import syntax there. * Ignore worker/ during linting I wasn't able to fix the following errors: /home/runner/work/stacker.news/stacker.news/worker/auction.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/auction.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/earn.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/earn.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/index.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/index.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/nostr.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/nostr.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/ots.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/ots.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/repin.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/repin.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/search.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/search.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/streak.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/streak.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/trust.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/trust.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/views.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/views.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) /home/runner/work/stacker.news/stacker.news/worker/wallet.js:0:0: Parsing error: No Babel config file detected for /home/runner/work/stacker.news/stacker.news/worker/wallet.js. Either disable config file checking with requireConfigFile: false, or configure Babel so that it can find the config files. (null) I tried to tell babel where to find the babel config file (.babelrc), specifying the babel config in worker/package.json under "babel", using babel.config.json etc. to no avail. However, afaict, we don't need babel for the worker since it won't run in a browser. Babel is only used to transpile code to target browsers. But it still would be nice to lint the worker code with standard. But we can figure this out later. * Fix worker imports from lib/ and api/ This fixes the issue that we can't use `{ "type": "module" }` in the root package.json since it breaks the app with this error: app | TypeError: next_auth_providers_credentials__WEBPACK_IMPORTED_MODULE_2__ is not a function app | at eval (webpack-internal:///./pages/api/auth/[...nextauth].js:218:20) app | at process.processTicksAndRejections (node:internal/process/task_queues:95:5) app | LND GRPC connection successful app | - error pages/api/auth/[...nextauth].js (139:2) @ CredentialsProvider app | - error Error [TypeError]: next_auth_providers_credentials__WEBPACK_IMPORTED_MODULE_2__ is not a function app | at eval (webpack-internal:///./pages/api/auth/[...nextauth].js:218:20) { app | digest: undefined app | } app | 137 | app | 138 | const providers = [ app | > 139 | CredentialsProvider({ app | | ^ app | 140 | id: 'lightning', app | 141 | name: 'Lightning', app | 142 | credentials: { app | TypeError: next_auth_providers_credentials__WEBPACK_IMPORTED_MODULE_2__ is not a function app | at eval (webpack-internal:///./pages/api/auth/[...nextauth].js:218:20) app | at process.processTicksAndRejections (node:internal/process/task_queues:95:5) build but we need to tell the worker that the files are MJS, else we get this error: worker | file:///app/worker/wallet.js:3 worker | import { datePivot } from '../lib/time.js' worker | ^^^^^^^^^ worker | SyntaxError: Named export 'datePivot' not found. The requested module '../lib/time.js' is a CommonJS module, which may not support all module.exports as named exports. worker | CommonJS modules can always be imported via the default export, for example using: worker | worker | import pkg from '../lib/time.js'; worker | const { datePivot } = pkg; worker | worker | at ModuleJob._instantiate (node:internal/modules/esm/module_job:124:21) worker | at async ModuleJob.run (node:internal/modules/esm/module_job:190:5) worker | worker | Node.js v18.17.0 worker | worker exited with code 1 * Fix global not defined in browser context * Also ignore api/ and lib/ during linting I did not want to do this but I was not able to fix this error in any other way I tried: /home/ekzyis/programming/stacker.news/api/lnd/index.js:0:0: Parsing error: No Babel config file detected for /home/ekzyis/programming/stacker.news/api/lnd/index.js. Either disable config file checking with requ ireConfigFile: false, or configure Babel so that it can find the config files. (null) Did not want to look deeper into all this standard, eslint, babel configuration stuff ... --------- Co-authored-by: ekzyis <ek@stacker.news> Co-authored-by: Keyan <34140557+huumn@users.noreply.github.com>
2023-09-24 01:19:35 +00:00
import serialize from '../api/resolvers/serial.js'
2023-10-13 19:30:58 +00:00
import { sendUserNotification } from '../api/webPush/index.js'
2023-12-04 15:35:54 +00:00
import { ANON_USER_ID, SN_USER_IDS } from '../lib/constants.js'
2023-10-13 19:30:58 +00:00
import { msatsToSats, numWithUnits } from '../lib/format.js'
import { PrismaClient } from '@prisma/client'
2022-03-17 20:13:19 +00:00
const ITEM_EACH_REWARD = 4.0
const UPVOTE_EACH_REWARD = 4.0
2023-09-13 23:36:42 +00:00
const TOP_PERCENTILE = 33
2023-08-10 02:27:53 +00:00
const TOTAL_UPPER_BOUND_MSATS = 1000000000
2022-07-07 19:14:22 +00:00
export async function earn ({ name }) {
2023-11-21 23:32:22 +00:00
// rewards are calculated sitewide still
// however for user gen subs currently only 50% of their fees go to rewards
// the other 50% goes to the founder of the sub
2022-03-17 20:13:19 +00:00
// grab a greedy connection
const models = new PrismaClient()
try {
2023-11-21 23:32:22 +00:00
// compute how much sn earned today
const [{ sum: sumDecimal }] = await models.$queryRaw`
2023-08-10 02:27:53 +00:00
SELECT coalesce(sum(msats), 0) as sum
FROM (
2023-11-21 23:32:22 +00:00
(SELECT ("ItemAct".msats - COALESCE("ReferralAct".msats, 0)) * COALESCE("Sub"."rewardsPct", 100) * 0.01 as msats
2023-08-10 02:27:53 +00:00
FROM "ItemAct"
2023-11-21 23:32:22 +00:00
JOIN "Item" ON "Item"."id" = "ItemAct"."itemId"
LEFT JOIN "Sub" ON "Sub"."name" = "Item"."subName"
2023-08-10 02:27:53 +00:00
LEFT JOIN "ReferralAct" ON "ReferralAct"."itemActId" = "ItemAct".id
2023-09-13 23:36:42 +00:00
WHERE date_trunc('day', "ItemAct".created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/Chicago') = date_trunc('day', (now() - interval '1 day') AT TIME ZONE 'America/Chicago')
AND "ItemAct".act <> 'TIP')
2023-08-10 02:27:53 +00:00
UNION ALL
(SELECT sats * 1000 as msats
FROM "Donation"
WHERE date_trunc('day', created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/Chicago') = date_trunc('day', (now() - interval '1 day') AT TIME ZONE 'America/Chicago'))
2023-08-14 19:37:09 +00:00
UNION ALL
multiple forwards on a post (#403) * multiple forwards on a post first phase of the multi-forward support * update the graphql mutation for discussion posts to accept and validate multiple forwards * update the discussion form to allow multiple forwards in the UI * start working on db schema changes * uncomment db schema, add migration to create the new model, and update create_item, update_item stored procedures * Propagate updates from discussion to poll, link, and bounty forms Update the create, update poll sql functions for multi forward support * Update gql, typedefs, and resolver to return forwarded users in items responses * UI changes to show multiple forward recipients, and conditional upvote logic changes * Update notification text to reflect multiple forwards upon vote action * Disallow duplicate stacker entries * reduce duplication in populating adv-post-form initial values * Update item_act sql function to implement multi-way forwarding * Update referral functions to scale referral bonuses for forwarded users * Update notification text to reflect non-100% forwarded sats cases * Update wallet history sql queries to accommodate multi-forward use cases * Block zaps for posts you are forwarded zaps at the API layer, in addition to in the UI * Delete fwdUserId column from Item table as part of migration * Fix how we calculate stacked sats after partial forwards in wallet history * Exclude entries from wallet history that are 0 stacked sats from posts with 100% forwarded to other users * Fix wallet history query for forwarded stacked sats to be scaled by the fwd pct * Reduce duplication in adv post form, and do some style tweaks for better layout * Use MAX_FORWARDS constants * Address various PR feedback * first enhancement pass * enhancement pass too --------- Co-authored-by: keyan <keyan.kousha+huumn@gmail.com> Co-authored-by: Keyan <34140557+huumn@users.noreply.github.com>
2023-08-23 22:44:17 +00:00
-- any earnings from anon's stack that are not forwarded to other users
2023-08-14 19:37:09 +00:00
(SELECT "ItemAct".msats
multiple forwards on a post (#403) * multiple forwards on a post first phase of the multi-forward support * update the graphql mutation for discussion posts to accept and validate multiple forwards * update the discussion form to allow multiple forwards in the UI * start working on db schema changes * uncomment db schema, add migration to create the new model, and update create_item, update_item stored procedures * Propagate updates from discussion to poll, link, and bounty forms Update the create, update poll sql functions for multi forward support * Update gql, typedefs, and resolver to return forwarded users in items responses * UI changes to show multiple forward recipients, and conditional upvote logic changes * Update notification text to reflect multiple forwards upon vote action * Disallow duplicate stacker entries * reduce duplication in populating adv-post-form initial values * Update item_act sql function to implement multi-way forwarding * Update referral functions to scale referral bonuses for forwarded users * Update notification text to reflect non-100% forwarded sats cases * Update wallet history sql queries to accommodate multi-forward use cases * Block zaps for posts you are forwarded zaps at the API layer, in addition to in the UI * Delete fwdUserId column from Item table as part of migration * Fix how we calculate stacked sats after partial forwards in wallet history * Exclude entries from wallet history that are 0 stacked sats from posts with 100% forwarded to other users * Fix wallet history query for forwarded stacked sats to be scaled by the fwd pct * Reduce duplication in adv post form, and do some style tweaks for better layout * Use MAX_FORWARDS constants * Address various PR feedback * first enhancement pass * enhancement pass too --------- Co-authored-by: keyan <keyan.kousha+huumn@gmail.com> Co-authored-by: Keyan <34140557+huumn@users.noreply.github.com>
2023-08-23 22:44:17 +00:00
FROM "Item"
JOIN "ItemAct" ON "ItemAct"."itemId" = "Item".id
LEFT JOIN "ItemForward" ON "ItemForward"."itemId" = "Item".id
WHERE "Item"."userId" = ${ANON_USER_ID} AND "ItemAct".act = 'TIP'
AND date_trunc('day', "ItemAct".created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/Chicago') = date_trunc('day', (now() - interval '1 day') AT TIME ZONE 'America/Chicago')
GROUP BY "ItemAct".id, "ItemAct".msats
HAVING COUNT("ItemForward".id) = 0)
2023-08-10 02:27:53 +00:00
) subquery`
// XXX primsa will return a Decimal (https://mikemcl.github.io/decimal.js)
// because sum of a BIGINT returns a NUMERIC type (https://www.postgresql.org/docs/13/functions-aggregate.html)
// and Decimal is what prisma maps it to https://www.prisma.io/docs/concepts/components/prisma-client/raw-database-access#raw-query-type-mapping
// so check it before coercing to Number
if (!sumDecimal || sumDecimal.lessThanOrEqualTo(0)) {
console.log('done', name, 'no sats to award today')
return
}
2023-08-01 16:10:07 +00:00
// extra sanity check on rewards ... if it's more than upper bound, we
// probably have a bug somewhere or we've grown A LOT
if (sumDecimal.greaterThan(TOTAL_UPPER_BOUND_MSATS)) {
console.log('done', name, 'error: too many sats to award today', sumDecimal)
return
}
2023-08-01 16:10:07 +00:00
const sum = Number(sumDecimal)
const heads = Math.random() < 0.5
// if this category is selected, double its proportion
// if it isn't select, zero its proportion
const itemRewardMult = heads ? 0 : 2.0
const upvoteRewardMult = heads ? 2.0 : 0
2023-08-10 02:27:53 +00:00
console.log(name, 'giving away', sum, 'msats', 'rewarding', heads ? 'items' : 'upvotes')
2022-12-08 00:04:02 +00:00
/*
2023-07-06 16:10:44 +00:00
How earnings (used to) work:
1/3: top 21% posts over last 36 hours, scored on a relative basis
1/3: top 21% comments over last 36 hours, scored on a relative basis
2022-07-07 19:14:22 +00:00
1/3: top upvoters of top posts/comments, scored on:
- their trust
- how much they tipped
- how early they upvoted it
- how the post/comment scored
2023-07-06 16:10:44 +00:00
2023-10-04 20:56:18 +00:00
Now: 100% of earnings go to either top 33% of comments/posts or top 33% of upvoters
2022-07-07 19:14:22 +00:00
*/
2022-03-18 12:29:02 +00:00
// get earners { userId, id, type, rank, proportion }
const earners = await models.$queryRaw`
2023-08-10 02:27:53 +00:00
-- get top 21% of posts and comments
2022-07-07 19:14:22 +00:00
WITH item_ratios AS (
SELECT *,
CASE WHEN "parentId" IS NULL THEN 'POST' ELSE 'COMMENT' END as type,
CASE WHEN "weightedVotes" > 0 THEN "weightedVotes"/(sum("weightedVotes") OVER (PARTITION BY "parentId" IS NULL)) ELSE 0 END AS ratio
FROM (
SELECT *,
2022-11-23 18:12:09 +00:00
NTILE(100) OVER (PARTITION BY "parentId" IS NULL ORDER BY ("weightedVotes"-"weightedDownVotes") desc) AS percentile,
ROW_NUMBER() OVER (PARTITION BY "parentId" IS NULL ORDER BY ("weightedVotes"-"weightedDownVotes") desc) AS rank
FROM
"Item"
WHERE created_at >= now_utc() - interval '36 hours'
2023-01-12 23:53:09 +00:00
AND "weightedVotes" > 0 AND "deletedAt" IS NULL AND NOT bio
) x
WHERE x.percentile <= ${TOP_PERCENTILE}
),
2023-08-10 02:27:53 +00:00
-- get top upvoters of top posts and comments
upvoter_islands AS (
SELECT "ItemAct"."userId", item_ratios.id, item_ratios.ratio, item_ratios."parentId",
"ItemAct".msats as tipped, "ItemAct".created_at as acted_at,
ROW_NUMBER() OVER (partition by item_ratios.id order by "ItemAct".created_at asc)
- ROW_NUMBER() OVER (partition by item_ratios.id, "ItemAct"."userId" order by "ItemAct".created_at asc) AS island
FROM item_ratios
JOIN "ItemAct" on "ItemAct"."itemId" = item_ratios.id
WHERE act = 'TIP'
),
-- isolate contiguous upzaps from the same user on the same item so that when we take the log
-- of the upzaps it accounts for successive zaps and does not disproporionately reward them
2022-07-07 19:14:22 +00:00
upvoters AS (
SELECT "userId", id, ratio, "parentId", GREATEST(log(sum(tipped) / 1000), 0) as tipped, min(acted_at) as acted_at
2023-08-10 02:27:53 +00:00
FROM upvoter_islands
GROUP BY "userId", id, ratio, "parentId", island
2022-07-07 19:14:22 +00:00
),
2023-08-10 02:27:53 +00:00
-- the relative contribution of each upvoter to the post/comment
-- early multiplier: 10/ln(early_rank + e)
-- we also weight by trust in a step wise fashion
2022-07-07 19:14:22 +00:00
upvoter_ratios AS (
2023-12-04 15:35:54 +00:00
SELECT "userId", sum(early_multiplier*tipped_ratio*ratio*CASE WHEN users.id = ANY (${SN_USER_IDS}) THEN 0.2 ELSE CEIL(users.trust*2)+1 END) as upvoter_ratio,
"parentId" IS NULL as "isPost", CASE WHEN "parentId" IS NULL THEN 'TIP_POST' ELSE 'TIP_COMMENT' END as type
2022-07-07 19:14:22 +00:00
FROM (
SELECT *,
2023-08-10 02:27:53 +00:00
10.0/LN(ROW_NUMBER() OVER (partition by id order by acted_at asc) + EXP(1.0)) AS early_multiplier,
2022-07-07 19:14:22 +00:00
tipped::float/(sum(tipped) OVER (partition by id)) tipped_ratio
FROM upvoters
) u
JOIN users on "userId" = users.id
GROUP BY "userId", "parentId" IS NULL
2023-10-04 20:56:18 +00:00
),
proportions AS (
SELECT "userId", NULL as id, type, ROW_NUMBER() OVER (PARTITION BY "isPost" ORDER BY upvoter_ratio DESC) as rank,
${itemRewardMult}*upvoter_ratio/(sum(upvoter_ratio) OVER (PARTITION BY "isPost"))/${UPVOTE_EACH_REWARD} as proportion
FROM upvoter_ratios
WHERE upvoter_ratio > 0
UNION ALL
SELECT "userId", id, type, rank, ${upvoteRewardMult}*ratio/${ITEM_EACH_REWARD} as proportion
FROM item_ratios)
SELECT "userId", id, type, rank, proportion
FROM proportions
2023-11-12 16:13:20 +00:00
WHERE proportion > 0.000001`
// in order to group earnings for users we use the same createdAt time for
// all earnings
const now = new Date(new Date().getTime())
// this is just a sanity check because it seems like a good idea
let total = 0
2022-03-18 12:29:02 +00:00
const notifications = {}
for (const earner of earners) {
const earnings = Math.floor(parseFloat(earner.proportion) * sum)
total += earnings
if (total > sum) {
console.log(name, 'total exceeds sum', total, '>', sum)
return
}
console.log('stacker', earner.userId, 'earned', earnings, 'proportion', earner.proportion, 'rank', earner.rank, 'type', earner.type)
2023-08-01 16:10:07 +00:00
if (earnings > 0) {
await serialize(models,
models.$executeRaw`SELECT earn(${earner.userId}::INTEGER, ${earnings},
2023-07-27 00:18:42 +00:00
${now}::timestamp without time zone, ${earner.type}::"EarnType", ${earner.id}::INTEGER, ${earner.rank}::INTEGER)`)
const userN = notifications[earner.userId] || {}
// sum total
const prevMsats = userN.msats || 0
const msats = earnings + prevMsats
// sum total per earn type (POST, COMMENT, TIP_COMMENT, TIP_POST)
const prevEarnTypeMsats = userN[earner.type]?.msats || 0
const earnTypeMsats = earnings + prevEarnTypeMsats
// best (=lowest) rank per earn type
const prevEarnTypeBestRank = userN[earner.type]?.bestRank
const earnTypeBestRank = prevEarnTypeBestRank ? Math.min(prevEarnTypeBestRank, Number(earner.rank)) : Number(earner.rank)
notifications[earner.userId] = {
...userN,
msats,
[earner.type]: { msats: earnTypeMsats, bestRank: earnTypeBestRank }
}
2022-03-18 12:29:02 +00:00
}
2023-10-13 19:30:58 +00:00
}
await territoryRevenue({ models })
2022-03-18 12:29:02 +00:00
Promise.allSettled(Object.entries(notifications).map(([userId, earnings]) =>
sendUserNotification(parseInt(userId, 10), buildUserNotification(earnings))
)).catch(console.error)
} finally {
models.$disconnect().catch(console.error)
}
2023-11-21 23:32:22 +00:00
}
async function territoryRevenue ({ models }) {
await serialize(models,
models.$executeRaw`
WITH revenue AS (
SELECT coalesce(sum(msats), 0) as revenue, "subName", "userId"
FROM (
SELECT ("ItemAct".msats - COALESCE("ReferralAct".msats, 0)) * (1 - (COALESCE("Sub"."rewardsPct", 100) * 0.01)) as msats,
"Sub"."name" as "subName", "Sub"."userId" as "userId"
FROM "ItemAct"
JOIN "Item" ON "Item"."id" = "ItemAct"."itemId"
JOIN "Sub" ON "Sub"."name" = "Item"."subName"
LEFT JOIN "ReferralAct" ON "ReferralAct"."itemActId" = "ItemAct".id
WHERE date_trunc('day', "ItemAct".created_at AT TIME ZONE 'UTC' AT TIME ZONE 'America/Chicago') = date_trunc('day', (now() - interval '1 day') AT TIME ZONE 'America/Chicago')
AND "ItemAct".act <> 'TIP'
AND "Sub".status <> 'STOPPED'
) subquery
GROUP BY "subName", "userId"
),
"SubActResult" AS (
INSERT INTO "SubAct" (msats, "subName", "userId", type)
SELECT revenue, "subName", "userId", 'REVENUE'
FROM revenue
WHERE revenue > 1000
RETURNING *
)
UPDATE users SET msats = users.msats + "SubActResult".msats
FROM "SubActResult"
WHERE users.id = "SubActResult"."userId"`
)
2022-03-18 12:29:02 +00:00
}
2023-10-13 19:30:58 +00:00
function buildUserNotification (earnings) {
const fmt = msats => numWithUnits(msatsToSats(msats, { abbreviate: false }))
const title = `you stacked ${fmt(earnings.msats)} in rewards`
2023-10-13 19:30:58 +00:00
const tag = 'EARN'
let body = ''
if (earnings.POST) body += `#${earnings.POST.bestRank} among posts with ${fmt(earnings.POST.msats)} in total\n`
if (earnings.COMMENT) body += `#${earnings.COMMENT.bestRank} among comments with ${fmt(earnings.COMMENT.msats)} in total\n`
if (earnings.TIP_POST) body += `#${earnings.TIP_POST.bestRank} in post zapping with ${fmt(earnings.TIP_POST.msats)} in total\n`
if (earnings.TIP_COMMENT) body += `#${earnings.TIP_COMMENT.bestRank} in comment zapping with ${fmt(earnings.TIP_COMMENT.msats)} in total`
2023-10-13 19:30:58 +00:00
return { title, tag, body }
}