From b672d015e209bcad24c4ef4666d61cdf6415dcc1 Mon Sep 17 00:00:00 2001 From: Keyan <34140557+huumn@users.noreply.github.com> Date: Sat, 15 Mar 2025 08:11:33 -0500 Subject: [PATCH] territory specific trust (#1965) * territory specific trust * functional parity with master * revert back to materialized view for ranking * update query for populating subWeightedVotes * fix anon hot comments * fix zap denormalization, change weightedComments to be for zaps, order updates of ancestors to prevent deadlocks * reduce weight of comment zaps for hot score * do zap ancestor updates together * initialize trust in new/unpopular territories * simplify denormalization of zap/downzaps * recompute all scores --- api/paidAction/downZap.js | 46 ++- api/paidAction/itemCreate.js | 11 +- api/paidAction/lib/territory.js | 27 ++ api/paidAction/territoryCreate.js | 9 +- api/paidAction/territoryUnarchive.js | 9 +- api/paidAction/zap.js | 61 ++-- api/resolvers/item.js | 82 ++---- .../migration.sql | 109 +++++++ .../migration.sql | 224 +++++++++++++++ prisma/schema.prisma | 177 +++++++----- worker/trust.js | 272 +++++++++++------- worker/views.js | 2 +- 12 files changed, 735 insertions(+), 294 deletions(-) create mode 100644 api/paidAction/lib/territory.js create mode 100644 prisma/migrations/20250308234113_user_sub_trust/migration.sql create mode 100644 prisma/migrations/20250311210455_comments_ranking_changes/migration.sql diff --git a/api/paidAction/downZap.js b/api/paidAction/downZap.js index f10bc17c..e6b82be2 100644 --- a/api/paidAction/downZap.js +++ b/api/paidAction/downZap.js @@ -1,5 +1,6 @@ import { PAID_ACTION_PAYMENT_METHODS } from '@/lib/constants' import { msatsToSats, satsToMsats } from '@/lib/format' +import { Prisma } from '@prisma/client' export const anonable = false @@ -48,9 +49,9 @@ export async function onPaid ({ invoice, actId }, { tx }) { let itemAct if (invoice) { await tx.itemAct.updateMany({ where: { invoiceId: invoice.id }, data: { invoiceActionState: 'PAID' } }) - itemAct = await tx.itemAct.findFirst({ where: { invoiceId: invoice.id } }) + itemAct = await tx.itemAct.findFirst({ where: { invoiceId: invoice.id }, include: { item: true } }) } else if (actId) { - itemAct = await tx.itemAct.findUnique({ where: { id: actId } }) + itemAct = await tx.itemAct.findUnique({ where: { id: actId }, include: { item: true } }) } else { throw new Error('No invoice or actId') } @@ -60,19 +61,34 @@ export async function onPaid ({ invoice, actId }, { tx }) { // denormalize downzaps await tx.$executeRaw` - WITH zapper AS ( - SELECT trust FROM users WHERE id = ${itemAct.userId}::INTEGER - ), zap AS ( - INSERT INTO "ItemUserAgg" ("userId", "itemId", "downZapSats") - VALUES (${itemAct.userId}::INTEGER, ${itemAct.itemId}::INTEGER, ${sats}::INTEGER) - ON CONFLICT ("itemId", "userId") DO UPDATE - SET "downZapSats" = "ItemUserAgg"."downZapSats" + ${sats}::INTEGER, updated_at = now() - RETURNING LOG("downZapSats" / GREATEST("downZapSats" - ${sats}::INTEGER, 1)::FLOAT) AS log_sats - ) - UPDATE "Item" - SET "weightedDownVotes" = "weightedDownVotes" + (zapper.trust * zap.log_sats) - FROM zap, zapper - WHERE "Item".id = ${itemAct.itemId}::INTEGER` + WITH territory AS ( + SELECT COALESCE(r."subName", i."subName", 'meta')::TEXT as "subName" + FROM "Item" i + LEFT JOIN "Item" r ON r.id = i."rootId" + WHERE i.id = ${itemAct.itemId}::INTEGER + ), zapper AS ( + SELECT + COALESCE(${itemAct.item.parentId + ? Prisma.sql`"zapCommentTrust"` + : Prisma.sql`"zapPostTrust"`}, 0) as "zapTrust", + COALESCE(${itemAct.item.parentId + ? Prisma.sql`"subZapCommentTrust"` + : Prisma.sql`"subZapPostTrust"`}, 0) as "subZapTrust" + FROM territory + LEFT JOIN "UserSubTrust" ust ON ust."subName" = territory."subName" + AND ust."userId" = ${itemAct.userId}::INTEGER + ), zap AS ( + INSERT INTO "ItemUserAgg" ("userId", "itemId", "downZapSats") + VALUES (${itemAct.userId}::INTEGER, ${itemAct.itemId}::INTEGER, ${sats}::INTEGER) + ON CONFLICT ("itemId", "userId") DO UPDATE + SET "downZapSats" = "ItemUserAgg"."downZapSats" + ${sats}::INTEGER, updated_at = now() + RETURNING LOG("downZapSats" / GREATEST("downZapSats" - ${sats}::INTEGER, 1)::FLOAT) AS log_sats + ) + UPDATE "Item" + SET "weightedDownVotes" = "weightedDownVotes" + zapper."zapTrust" * zap.log_sats, + "subWeightedDownVotes" = "subWeightedDownVotes" + zapper."subZapTrust" * zap.log_sats + FROM zap, zapper + WHERE "Item".id = ${itemAct.itemId}::INTEGER` } export async function onFail ({ invoice }, { tx }) { diff --git a/api/paidAction/itemCreate.js b/api/paidAction/itemCreate.js index b8e8f416..d6fb603f 100644 --- a/api/paidAction/itemCreate.js +++ b/api/paidAction/itemCreate.js @@ -252,15 +252,18 @@ export async function onPaid ({ invoice, id }, context) { JOIN users ON "Item"."userId" = users.id WHERE "Item".id = ${item.id}::INTEGER ), ancestors AS ( + SELECT "Item".* + FROM "Item", comment + WHERE "Item".path @> comment.path AND "Item".id <> comment.id + ORDER BY "Item".id + ), updated_ancestors AS ( UPDATE "Item" SET ncomments = "Item".ncomments + 1, "lastCommentAt" = GREATEST("Item"."lastCommentAt", comment.created_at), - "weightedComments" = "Item"."weightedComments" + - CASE WHEN comment."userId" = "Item"."userId" THEN 0 ELSE comment.trust END, "nDirectComments" = "Item"."nDirectComments" + CASE WHEN comment."parentId" = "Item".id THEN 1 ELSE 0 END - FROM comment - WHERE "Item".path @> comment.path AND "Item".id <> comment.id + FROM comment, ancestors + WHERE "Item".id = ancestors.id RETURNING "Item".* ) INSERT INTO "Reply" (created_at, updated_at, "ancestorId", "ancestorUserId", "itemId", "userId", level) diff --git a/api/paidAction/lib/territory.js b/api/paidAction/lib/territory.js new file mode 100644 index 00000000..849ff11c --- /dev/null +++ b/api/paidAction/lib/territory.js @@ -0,0 +1,27 @@ +import { USER_ID } from '@/lib/constants' + +export const GLOBAL_SEEDS = [USER_ID.k00b, USER_ID.ek] + +export function initialTrust ({ name, userId }) { + const results = GLOBAL_SEEDS.map(id => ({ + subName: name, + userId: id, + zapPostTrust: 1, + subZapPostTrust: 1, + zapCommentTrust: 1, + subZapCommentTrust: 1 + })) + + if (!GLOBAL_SEEDS.includes(userId)) { + results.push({ + subName: name, + userId, + zapPostTrust: 0, + subZapPostTrust: 1, + zapCommentTrust: 0, + subZapCommentTrust: 1 + }) + } + + return results +} diff --git a/api/paidAction/territoryCreate.js b/api/paidAction/territoryCreate.js index ef2610d5..a9316cdb 100644 --- a/api/paidAction/territoryCreate.js +++ b/api/paidAction/territoryCreate.js @@ -1,6 +1,7 @@ import { PAID_ACTION_PAYMENT_METHODS, TERRITORY_PERIOD_COST } from '@/lib/constants' import { satsToMsats } from '@/lib/format' import { nextBilling } from '@/lib/territory' +import { initialTrust } from './lib/territory' export const anonable = false @@ -20,7 +21,7 @@ export async function perform ({ invoiceId, ...data }, { me, cost, tx }) { const billedLastAt = new Date() const billPaidUntil = nextBilling(billedLastAt, billingType) - return await tx.sub.create({ + const sub = await tx.sub.create({ data: { ...data, billedLastAt, @@ -42,6 +43,12 @@ export async function perform ({ invoiceId, ...data }, { me, cost, tx }) { } } }) + + await tx.userSubTrust.createMany({ + data: initialTrust({ name: sub.name, userId: sub.userId }) + }) + + return sub } export async function describe ({ name }) { diff --git a/api/paidAction/territoryUnarchive.js b/api/paidAction/territoryUnarchive.js index 95c63eb7..9f79963e 100644 --- a/api/paidAction/territoryUnarchive.js +++ b/api/paidAction/territoryUnarchive.js @@ -1,6 +1,7 @@ import { PAID_ACTION_PAYMENT_METHODS, TERRITORY_PERIOD_COST } from '@/lib/constants' import { satsToMsats } from '@/lib/format' import { nextBilling } from '@/lib/territory' +import { initialTrust } from './lib/territory' export const anonable = false @@ -65,7 +66,7 @@ export async function perform ({ name, invoiceId, ...data }, { me, cost, tx }) { } }) - return await tx.sub.update({ + const updatedSub = await tx.sub.update({ data, // optimistic concurrency control // make sure none of the relevant fields have changed since we fetched the sub @@ -76,6 +77,12 @@ export async function perform ({ name, invoiceId, ...data }, { me, cost, tx }) { } } }) + + await tx.userSubTrust.createMany({ + data: initialTrust({ name: updatedSub.name, userId: updatedSub.userId }) + }) + + return updatedSub } export async function describe ({ name }, context) { diff --git a/api/paidAction/zap.js b/api/paidAction/zap.js index 9e2b4e22..dea0d057 100644 --- a/api/paidAction/zap.js +++ b/api/paidAction/zap.js @@ -2,6 +2,7 @@ import { PAID_ACTION_PAYMENT_METHODS, USER_ID } from '@/lib/constants' import { msatsToSats, satsToMsats } from '@/lib/format' import { notifyZapped } from '@/lib/webPush' import { getInvoiceableWallets } from '@/wallets/server' +import { Prisma } from '@prisma/client' export const anonable = true @@ -149,8 +150,22 @@ export async function onPaid ({ invoice, actIds }, { tx }) { // perform denomormalized aggregates: weighted votes, upvotes, msats, lastZapAt // NOTE: for the rows that might be updated by a concurrent zap, we use UPDATE for implicit locking await tx.$queryRaw` - WITH zapper AS ( - SELECT trust FROM users WHERE id = ${itemAct.userId}::INTEGER + WITH territory AS ( + SELECT COALESCE(r."subName", i."subName", 'meta')::TEXT as "subName" + FROM "Item" i + LEFT JOIN "Item" r ON r.id = i."rootId" + WHERE i.id = ${itemAct.itemId}::INTEGER + ), zapper AS ( + SELECT + COALESCE(${itemAct.item.parentId + ? Prisma.sql`"zapCommentTrust"` + : Prisma.sql`"zapPostTrust"`}, 0) as "zapTrust", + COALESCE(${itemAct.item.parentId + ? Prisma.sql`"subZapCommentTrust"` + : Prisma.sql`"subZapPostTrust"`}, 0) as "subZapTrust" + FROM territory + LEFT JOIN "UserSubTrust" ust ON ust."subName" = territory."subName" + AND ust."userId" = ${itemAct.userId}::INTEGER ), zap AS ( INSERT INTO "ItemUserAgg" ("userId", "itemId", "zapSats") VALUES (${itemAct.userId}::INTEGER, ${itemAct.itemId}::INTEGER, ${sats}::INTEGER) @@ -158,17 +173,30 @@ export async function onPaid ({ invoice, actIds }, { tx }) { SET "zapSats" = "ItemUserAgg"."zapSats" + ${sats}::INTEGER, updated_at = now() RETURNING ("zapSats" = ${sats}::INTEGER)::INTEGER as first_vote, LOG("zapSats" / GREATEST("zapSats" - ${sats}::INTEGER, 1)::FLOAT) AS log_sats + ), item_zapped AS ( + UPDATE "Item" + SET + "weightedVotes" = "weightedVotes" + zapper."zapTrust" * zap.log_sats, + "subWeightedVotes" = "subWeightedVotes" + zapper."subZapTrust" * zap.log_sats, + upvotes = upvotes + zap.first_vote, + msats = "Item".msats + ${msats}::BIGINT, + mcredits = "Item".mcredits + ${invoice?.invoiceForward ? 0n : msats}::BIGINT, + "lastZapAt" = now() + FROM zap, zapper + WHERE "Item".id = ${itemAct.itemId}::INTEGER + RETURNING "Item".*, zapper."zapTrust" * zap.log_sats as "weightedVote" + ), ancestors AS ( + SELECT "Item".* + FROM "Item", item_zapped + WHERE "Item".path @> item_zapped.path AND "Item".id <> item_zapped.id + ORDER BY "Item".id ) UPDATE "Item" - SET - "weightedVotes" = "weightedVotes" + (zapper.trust * zap.log_sats), - upvotes = upvotes + zap.first_vote, - msats = "Item".msats + ${msats}::BIGINT, - mcredits = "Item".mcredits + ${invoice?.invoiceForward ? 0n : msats}::BIGINT, - "lastZapAt" = now() - FROM zap, zapper - WHERE "Item".id = ${itemAct.itemId}::INTEGER - RETURNING "Item".*` + SET "weightedComments" = "Item"."weightedComments" + item_zapped."weightedVote", + "commentMsats" = "Item"."commentMsats" + ${msats}::BIGINT, + "commentMcredits" = "Item"."commentMcredits" + ${invoice?.invoiceForward ? 0n : msats}::BIGINT + FROM item_zapped, ancestors + WHERE "Item".id = ancestors.id` // record potential bounty payment // NOTE: we are at least guaranteed that we see the update "ItemUserAgg" from our tx so we can trust @@ -188,17 +216,6 @@ export async function onPaid ({ invoice, actIds }, { tx }) { SET "bountyPaidTo" = array_remove(array_append(array_remove("bountyPaidTo", bounty.target), bounty.target), NULL) FROM bounty WHERE "Item".id = bounty.id AND bounty.paid` - - // update commentMsats on ancestors - await tx.$executeRaw` - WITH zapped AS ( - SELECT * FROM "Item" WHERE id = ${itemAct.itemId}::INTEGER - ) - UPDATE "Item" - SET "commentMsats" = "Item"."commentMsats" + ${msats}::BIGINT, - "commentMcredits" = "Item"."commentMcredits" + ${invoice?.invoiceForward ? 0n : msats}::BIGINT - FROM zapped - WHERE "Item".path @> zapped.path AND "Item".id <> zapped.id` } export async function nonCriticalSideEffects ({ invoice, actIds }, { models }) { diff --git a/api/resolvers/item.js b/api/resolvers/item.js index 2ff08633..b1a53e54 100644 --- a/api/resolvers/item.js +++ b/api/resolvers/item.js @@ -39,16 +39,12 @@ function commentsOrderByClause (me, models, sort) { COALESCE("Item"."invoicePaidAt", "Item".created_at) DESC, "Item".id DESC` } - if (me && sort === 'hot') { + if (sort === 'hot') { return `ORDER BY ${sharedSorts}, - "personal_hot_score" DESC NULLS LAST, - "Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC` + "hotScore" DESC NULLS LAST, + "Item".msats DESC, "Item".id DESC` } else { - if (sort === 'top') { - return `ORDER BY ${sharedSorts}, ${orderByNumerator({ models, commentScaler: 0 })} DESC NULLS LAST, "Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC` - } else { - return `ORDER BY ${sharedSorts}, ${orderByNumerator({ models, commentScaler: 0, considerBoost: true })}/POWER(GREATEST(3, EXTRACT(EPOCH FROM (now_utc() - "Item".created_at))/3600), 1.3) DESC NULLS LAST, "Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC` - } + return `ORDER BY ${sharedSorts}, "Item"."weightedVotes" - "Item"."weightedDownVotes" DESC NULLS LAST, "Item".msats DESC, "Item".id DESC` } } @@ -138,14 +134,14 @@ export async function getAd (parent, { sub, subArr = [], showNsfw = false }, { m }, ...subArr))?.[0] || null } -const orderByClause = (by, me, models, type) => { +const orderByClause = (by, me, models, type, sub) => { switch (by) { case 'comments': return 'ORDER BY "Item".ncomments DESC' case 'sats': return 'ORDER BY "Item".msats DESC' case 'zaprank': - return topOrderByWeightedSats(me, models) + return topOrderByWeightedSats(me, models, sub) case 'boost': return 'ORDER BY "Item".boost DESC' case 'random': @@ -155,22 +151,8 @@ const orderByClause = (by, me, models, type) => { } } -export function orderByNumerator ({ models, commentScaler = 0.5, considerBoost = false }) { - return `((CASE WHEN "Item"."weightedVotes" - "Item"."weightedDownVotes" > 0 THEN - GREATEST("Item"."weightedVotes" - "Item"."weightedDownVotes", POWER("Item"."weightedVotes" - "Item"."weightedDownVotes", 1.2)) - ELSE - "Item"."weightedVotes" - "Item"."weightedDownVotes" - END + "Item"."weightedComments"*${commentScaler}) + ${considerBoost ? `("Item".boost / ${BOOST_MULT})` : 0})` -} - -export function joinZapRankPersonalView (me, models) { - let join = ` JOIN zap_rank_personal_view g ON g.id = "Item".id AND g."viewerId" = ${GLOBAL_SEED} ` - - if (me) { - join += ` LEFT JOIN zap_rank_personal_view l ON l.id = g.id AND l."viewerId" = ${me.id} ` - } - - return join +export function joinHotScoreView (me, models) { + return ' JOIN hot_score_view g ON g.id = "Item".id ' } // this grabs all the stuff we need to display the item list and only @@ -475,10 +457,10 @@ export default { await filterClause(me, models, type), by === 'boost' && '"Item".boost > 0', muteClause(me))} - ${orderByClause(by || 'zaprank', me, models, type)} + ${orderByClause(by || 'zaprank', me, models, type, sub)} OFFSET $3 LIMIT $4`, - orderBy: orderByClause(by || 'zaprank', me, models, type) + orderBy: orderByClause(by || 'zaprank', me, models, type, sub) }, ...whenRange(when, from, to || decodedCursor.time), decodedCursor.offset, limit, ...subArr) break case 'random': @@ -571,10 +553,10 @@ export default { me, models, query: ` - ${SELECT}, ${me ? 'GREATEST(g.tf_hot_score, l.tf_hot_score)' : 'g.tf_hot_score'} AS rank + ${SELECT}, g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore" FROM "Item" LEFT JOIN "Sub" ON "Sub"."name" = "Item"."subName" - ${joinZapRankPersonalView(me, models)} + ${joinHotScoreView(me, models)} ${whereClause( // in home (sub undefined), filter out global pinned items since we inject them later sub ? '"Item"."pinId" IS NULL' : 'NOT ("Item"."pinId" IS NOT NULL AND "Item"."subName" IS NULL)', @@ -587,40 +569,11 @@ export default { await filterClause(me, models, type), subClause(sub, 3, 'Item', me, showNsfw), muteClause(me))} - ORDER BY rank DESC + ORDER BY ${sub ? '"subHotScore"' : '"hotScore"'} DESC, "Item".msats DESC, "Item".id DESC OFFSET $1 LIMIT $2`, - orderBy: 'ORDER BY rank DESC' + orderBy: `ORDER BY ${sub ? '"subHotScore"' : '"hotScore"'} DESC, "Item".msats DESC, "Item".id DESC` }, decodedCursor.offset, limit, ...subArr) - - // XXX this is mostly for subs that are really empty - if (items.length < limit) { - items = await itemQueryWithMeta({ - me, - models, - query: ` - ${SELECT} - FROM "Item" - LEFT JOIN "Sub" ON "Sub"."name" = "Item"."subName" - ${whereClause( - subClause(sub, 3, 'Item', me, showNsfw), - muteClause(me), - // in home (sub undefined), filter out global pinned items since we inject them later - sub ? '"Item"."pinId" IS NULL' : 'NOT ("Item"."pinId" IS NOT NULL AND "Item"."subName" IS NULL)', - '"Item"."deletedAt" IS NULL', - '"Item"."parentId" IS NULL', - '"Item".bio = false', - ad ? `"Item".id <> ${ad.id}` : '', - activeOrMine(me), - await filterClause(me, models, type))} - ORDER BY ${orderByNumerator({ models, considerBoost: true })}/POWER(GREATEST(3, EXTRACT(EPOCH FROM (now_utc() - "Item".created_at))/3600), 1.3) DESC NULLS LAST, - "Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC - OFFSET $1 - LIMIT $2`, - orderBy: `ORDER BY ${orderByNumerator({ models, considerBoost: true })}/POWER(GREATEST(3, EXTRACT(EPOCH FROM (now_utc() - "Item".created_at))/3600), 1.3) DESC NULLS LAST, - "Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC` - }, decodedCursor.offset, limit, ...subArr) - } break } break @@ -1574,6 +1527,9 @@ export const SELECT = `SELECT "Item".*, "Item".created_at as "createdAt", "Item".updated_at as "updatedAt", ltree2text("Item"."path") AS "path"` -function topOrderByWeightedSats (me, models) { - return `ORDER BY ${orderByNumerator({ models })} DESC NULLS LAST, "Item".id DESC` +function topOrderByWeightedSats (me, models, sub) { + if (sub) { + return 'ORDER BY "Item"."subWeightedVotes" - "Item"."subWeightedDownVotes" DESC, "Item".msats DESC, "Item".id DESC' + } + return 'ORDER BY "Item"."weightedVotes" - "Item"."weightedDownVotes" DESC, "Item".msats DESC, "Item".id DESC' } diff --git a/prisma/migrations/20250308234113_user_sub_trust/migration.sql b/prisma/migrations/20250308234113_user_sub_trust/migration.sql new file mode 100644 index 00000000..bb73253a --- /dev/null +++ b/prisma/migrations/20250308234113_user_sub_trust/migration.sql @@ -0,0 +1,109 @@ +-- AlterTable +ALTER TABLE "Item" ADD COLUMN "subWeightedDownVotes" FLOAT NOT NULL DEFAULT 0, +ADD COLUMN "subWeightedVotes" FLOAT NOT NULL DEFAULT 0; + +CREATE INDEX "Item.sumSubVotes_index" ON "Item"(("subWeightedVotes" - "subWeightedDownVotes")); + +-- CreateTable +CREATE TABLE "UserSubTrust" ( + "subName" CITEXT NOT NULL, + "userId" INTEGER NOT NULL, + "zapPostTrust" DOUBLE PRECISION NOT NULL DEFAULT 0, + "subZapPostTrust" DOUBLE PRECISION NOT NULL DEFAULT 0, + "zapCommentTrust" DOUBLE PRECISION NOT NULL DEFAULT 0, + "subZapCommentTrust" DOUBLE PRECISION NOT NULL DEFAULT 0, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "UserSubTrust_pkey" PRIMARY KEY ("userId","subName") +); + +-- AddForeignKey +ALTER TABLE "UserSubTrust" ADD CONSTRAINT "UserSubTrust_userId_fkey" FOREIGN KEY ("userId") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "UserSubTrust" ADD CONSTRAINT "UserSubTrust_subName_fkey" FOREIGN KEY ("subName") REFERENCES "Sub"("name") ON DELETE CASCADE ON UPDATE CASCADE; + +-- UserSubTrust is NOT populated, so this is a no-op ... but useful having it written out for migrating manually on deployment +UPDATE "Item" +SET "subWeightedVotes" = subquery."subWeightedVotes", + "subWeightedDownVotes" = subquery."subWeightedDownVotes", + "weightedVotes" = subquery."weightedVotes", + "weightedDownVotes" = subquery."weightedDownVotes" +FROM ( + WITH sub_votes AS ( + SELECT "ItemAct"."itemId", + CASE WHEN (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('TIP', 'FEE')))::FLOAT > 0 + THEN COALESCE( + LOG( + (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('TIP', 'FEE')))::FLOAT / 1000 + ) * CASE + WHEN "Item"."parentId" IS NULL + THEN "UserSubTrust"."subZapPostTrust" + ELSE "UserSubTrust"."subZapCommentTrust" + END, 0) + ELSE 0 + END AS "subWeightedVotes", + CASE WHEN (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('TIP', 'FEE')))::FLOAT > 0 + THEN COALESCE( + LOG( + (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('TIP', 'FEE')))::FLOAT / 1000 + ) * CASE + WHEN "Item"."parentId" IS NULL + THEN "UserSubTrust"."zapPostTrust" + ELSE "UserSubTrust"."zapCommentTrust" + END, 0) + ELSE 0 + END AS "weightedVotes", + CASE WHEN (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('DONT_LIKE_THIS')))::FLOAT > 0 + THEN COALESCE( + LOG( + (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('DONT_LIKE_THIS')))::FLOAT / 1000 + ) * CASE + WHEN "Item"."parentId" IS NULL + THEN "UserSubTrust"."subZapPostTrust" + ELSE "UserSubTrust"."subZapCommentTrust" + END, 0) + ELSE 0 + END AS "subWeightedDownVotes", + CASE WHEN (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('DONT_LIKE_THIS')))::FLOAT > 0 + THEN COALESCE( + LOG( + (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('DONT_LIKE_THIS')))::FLOAT / 1000 + ) * CASE + WHEN "Item"."parentId" IS NULL + THEN "UserSubTrust"."zapPostTrust" + ELSE "UserSubTrust"."zapCommentTrust" + END, 0) + ELSE 0 + END AS "weightedDownVotes" + FROM "ItemAct" + JOIN "UserSubTrust" ON "ItemAct"."userId" = "UserSubTrust"."userId" + JOIN "Item" ON "Item".id = "ItemAct"."itemId" + AND "UserSubTrust"."subName" = "Item"."subName" + AND "Item"."userId" <> "ItemAct"."userId" + WHERE "ItemAct".act IN ('TIP', 'FEE', 'DONT_LIKE_THIS') + GROUP BY "ItemAct"."itemId", "ItemAct"."userId", "UserSubTrust"."subZapPostTrust", + "UserSubTrust"."subZapCommentTrust", "UserSubTrust"."zapPostTrust", "UserSubTrust"."zapCommentTrust", + "Item"."parentId" + ) + SELECT "itemId", SUM("subWeightedVotes") AS "subWeightedVotes", SUM("subWeightedDownVotes") AS "subWeightedDownVotes", + SUM("weightedVotes") AS "weightedVotes", SUM("weightedDownVotes") AS "weightedDownVotes" + FROM sub_votes + GROUP BY "itemId" +) AS subquery +WHERE "Item".id = subquery."itemId"; + +CREATE MATERIALIZED VIEW IF NOT EXISTS hot_score_view AS + SELECT id, + ("Item"."weightedVotes" - "Item"."weightedDownVotes" + ("Item"."weightedComments"*0.25) + ("Item".boost / 5000)) + / POWER(GREATEST(3, EXTRACT(EPOCH FROM (now() - "Item".created_at))/3600), 1.1) AS hot_score, + ("Item"."subWeightedVotes" - "Item"."subWeightedDownVotes" + ("Item"."weightedComments"*0.25) + ("Item".boost / 5000)) + / POWER(GREATEST(3, EXTRACT(EPOCH FROM (now() - "Item".created_at))/3600), 1.1) AS sub_hot_score + FROM "Item" + WHERE "Item"."weightedVotes" > 0 OR "Item"."weightedDownVotes" > 0 OR "Item"."subWeightedVotes" > 0 + OR "Item"."subWeightedDownVotes" > 0 OR "Item"."weightedComments" > 0 OR "Item".boost > 0; + +CREATE UNIQUE INDEX IF NOT EXISTS hot_score_view_id_idx ON hot_score_view(id); +CREATE INDEX IF NOT EXISTS hot_score_view_hot_score_idx ON hot_score_view(hot_score DESC NULLS LAST); +CREATE INDEX IF NOT EXISTS hot_score_view_sub_hot_score_idx ON hot_score_view(sub_hot_score DESC NULLS LAST); diff --git a/prisma/migrations/20250311210455_comments_ranking_changes/migration.sql b/prisma/migrations/20250311210455_comments_ranking_changes/migration.sql new file mode 100644 index 00000000..f7975b83 --- /dev/null +++ b/prisma/migrations/20250311210455_comments_ranking_changes/migration.sql @@ -0,0 +1,224 @@ +-- add limit and offset +CREATE OR REPLACE FUNCTION item_comments_zaprank_with_me_limited( + _item_id int, _global_seed int, _me_id int, _limit int, _offset int, _grandchild_limit int, + _level int, _where text, _order_by text) + RETURNS jsonb + LANGUAGE plpgsql VOLATILE PARALLEL SAFE AS +$$ +DECLARE + result jsonb; +BEGIN + IF _level < 1 THEN + RETURN '[]'::jsonb; + END IF; + + EXECUTE 'CREATE TEMP TABLE IF NOT EXISTS t_item ON COMMIT DROP AS ' + || 'WITH RECURSIVE base AS ( ' + || ' (SELECT "Item".*, 1 as level, ROW_NUMBER() OVER () as rn ' + || ' FROM "Item" ' + || ' WHERE "Item"."parentId" = $1 ' + || _order_by || ' ' + || ' LIMIT $4 ' + || ' OFFSET $5) ' + || ' UNION ALL ' + || ' (SELECT "Item".*, b.level + 1, ROW_NUMBER() OVER (PARTITION BY "Item"."parentId" ' || _order_by || ') as rn ' + || ' FROM "Item" ' + || ' JOIN base b ON "Item"."parentId" = b.id ' + || ' LEFT JOIN hot_score_view g ON g.id = "Item".id ' + || ' WHERE b.level < $7 AND (b.level = 1 OR b.rn <= $6)) ' + || ') ' + || 'SELECT "Item".*, ' + || ' "Item".created_at at time zone ''UTC'' AS "createdAt", ' + || ' "Item".updated_at at time zone ''UTC'' AS "updatedAt", ' + || ' "Item"."invoicePaidAt" at time zone ''UTC'' AS "invoicePaidAtUTC", ' + || ' to_jsonb(users.*) || jsonb_build_object(''meMute'', "Mute"."mutedId" IS NOT NULL) AS user, ' + || ' COALESCE("ItemAct"."meMsats", 0) AS "meMsats", ' + || ' COALESCE("ItemAct"."mePendingMsats", 0) as "mePendingMsats", ' + || ' COALESCE("ItemAct"."meDontLikeMsats", 0) AS "meDontLikeMsats", ' + || ' COALESCE("ItemAct"."meMcredits", 0) AS "meMcredits", ' + || ' COALESCE("ItemAct"."mePendingMcredits", 0) as "mePendingMcredits", ' + || ' "Bookmark"."itemId" IS NOT NULL AS "meBookmark", ' + || ' "ThreadSubscription"."itemId" IS NOT NULL AS "meSubscription", ' + || ' g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore" ' + || 'FROM base "Item" ' + || 'JOIN users ON users.id = "Item"."userId" ' + || ' LEFT JOIN "Mute" ON "Mute"."muterId" = $3 AND "Mute"."mutedId" = "Item"."userId" ' + || ' LEFT JOIN "Bookmark" ON "Bookmark"."userId" = $3 AND "Bookmark"."itemId" = "Item".id ' + || ' LEFT JOIN "ThreadSubscription" ON "ThreadSubscription"."userId" = $3 AND "ThreadSubscription"."itemId" = "Item".id ' + || ' LEFT JOIN hot_score_view g ON g.id = "Item".id ' + || 'LEFT JOIN LATERAL ( ' + || ' SELECT "itemId", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND "InvoiceForward".id IS NOT NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "meMsats", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND "InvoiceForward".id IS NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "meMcredits", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS NOT DISTINCT FROM ''PENDING'' AND "InvoiceForward".id IS NOT NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "mePendingMsats", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS NOT DISTINCT FROM ''PENDING'' AND "InvoiceForward".id IS NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "mePendingMcredits", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND act = ''DONT_LIKE_THIS'') AS "meDontLikeMsats" ' + || ' FROM "ItemAct" ' + || ' LEFT JOIN "Invoice" ON "Invoice".id = "ItemAct"."invoiceId" ' + || ' LEFT JOIN "InvoiceForward" ON "InvoiceForward"."invoiceId" = "Invoice"."id" ' + || ' WHERE "ItemAct"."userId" = $3 ' + || ' AND "ItemAct"."itemId" = "Item".id ' + || ' GROUP BY "ItemAct"."itemId" ' + || ') "ItemAct" ON true ' + || 'WHERE ("Item".level = 1 OR "Item".rn <= $6 - "Item".level + 2) ' || _where || ' ' + USING _item_id, _global_seed, _me_id, _limit, _offset, _grandchild_limit, _level, _where, _order_by; + + EXECUTE '' + || 'SELECT COALESCE(jsonb_agg(sub), ''[]''::jsonb) AS comments ' + || 'FROM ( ' + || ' SELECT "Item".*, item_comments_zaprank_with_me_limited("Item".id, $2, $3, $4, $5, $6, $7 - 1, $8, $9) AS comments ' + || ' FROM t_item "Item" ' + || ' WHERE "Item"."parentId" = $1 ' + || _order_by + || ' ) sub' + INTO result USING _item_id, _global_seed, _me_id, _limit, _offset, _grandchild_limit, _level, _where, _order_by; + + RETURN result; +END +$$; + +-- add cowboy credits +CREATE OR REPLACE FUNCTION item_comments_zaprank_with_me(_item_id int, _global_seed int, _me_id int, _level int, _where text, _order_by text) + RETURNS jsonb + LANGUAGE plpgsql VOLATILE PARALLEL SAFE AS +$$ +DECLARE + result jsonb; +BEGIN + IF _level < 1 THEN + RETURN '[]'::jsonb; + END IF; + + EXECUTE 'CREATE TEMP TABLE IF NOT EXISTS t_item ON COMMIT DROP AS' + || ' SELECT "Item".*, "Item".created_at at time zone ''UTC'' AS "createdAt", "Item".updated_at at time zone ''UTC'' AS "updatedAt", ' + || ' "Item"."invoicePaidAt" at time zone ''UTC'' AS "invoicePaidAtUTC", to_jsonb(users.*) || jsonb_build_object(''meMute'', "Mute"."mutedId" IS NOT NULL) AS user, ' + || ' COALESCE("ItemAct"."meMsats", 0) AS "meMsats", COALESCE("ItemAct"."mePendingMsats", 0) as "mePendingMsats", COALESCE("ItemAct"."meDontLikeMsats", 0) AS "meDontLikeMsats", ' + || ' COALESCE("ItemAct"."meMcredits", 0) AS "meMcredits", COALESCE("ItemAct"."mePendingMcredits", 0) as "mePendingMcredits", ' + || ' "Bookmark"."itemId" IS NOT NULL AS "meBookmark", "ThreadSubscription"."itemId" IS NOT NULL AS "meSubscription", ' + || ' g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore" ' + || ' FROM "Item" ' + || ' JOIN users ON users.id = "Item"."userId" ' + || ' LEFT JOIN "Mute" ON "Mute"."muterId" = $5 AND "Mute"."mutedId" = "Item"."userId"' + || ' LEFT JOIN "Bookmark" ON "Bookmark"."userId" = $5 AND "Bookmark"."itemId" = "Item".id ' + || ' LEFT JOIN "ThreadSubscription" ON "ThreadSubscription"."userId" = $5 AND "ThreadSubscription"."itemId" = "Item".id ' + || ' LEFT JOIN LATERAL ( ' + || ' SELECT "itemId", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND "InvoiceForward".id IS NOT NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "meMsats", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND "InvoiceForward".id IS NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "meMcredits", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS NOT DISTINCT FROM ''PENDING'' AND "InvoiceForward".id IS NOT NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "mePendingMsats", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS NOT DISTINCT FROM ''PENDING'' AND "InvoiceForward".id IS NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "mePendingMcredits", ' + || ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND act = ''DONT_LIKE_THIS'') AS "meDontLikeMsats" ' + || ' FROM "ItemAct" ' + || ' LEFT JOIN "Invoice" ON "Invoice".id = "ItemAct"."invoiceId" ' + || ' LEFT JOIN "InvoiceForward" ON "InvoiceForward"."invoiceId" = "Invoice"."id" ' + || ' WHERE "ItemAct"."userId" = $5 ' + || ' AND "ItemAct"."itemId" = "Item".id ' + || ' GROUP BY "ItemAct"."itemId" ' + || ' ) "ItemAct" ON true ' + || ' LEFT JOIN hot_score_view g ON g.id = "Item".id ' + || ' WHERE "Item".path <@ (SELECT path FROM "Item" WHERE id = $1) ' || _where || ' ' + USING _item_id, _level, _where, _order_by, _me_id, _global_seed; + + EXECUTE '' + || 'SELECT COALESCE(jsonb_agg(sub), ''[]''::jsonb) AS comments ' + || 'FROM ( ' + || ' SELECT "Item".*, item_comments_zaprank_with_me("Item".id, $6, $5, $2 - 1, $3, $4) AS comments ' + || ' FROM t_item "Item" ' + || ' WHERE "Item"."parentId" = $1 ' + || _order_by + || ' ) sub' + INTO result USING _item_id, _level, _where, _order_by, _me_id, _global_seed; + + RETURN result; +END +$$; + +CREATE OR REPLACE FUNCTION item_comments(_item_id int, _level int, _where text, _order_by text) + RETURNS jsonb + LANGUAGE plpgsql VOLATILE PARALLEL SAFE AS +$$ +DECLARE + result jsonb; +BEGIN + IF _level < 1 THEN + RETURN '[]'::jsonb; + END IF; + + EXECUTE 'CREATE TEMP TABLE IF NOT EXISTS t_item ON COMMIT DROP AS' + || ' SELECT "Item".*, "Item".created_at at time zone ''UTC'' AS "createdAt", "Item".updated_at at time zone ''UTC'' AS "updatedAt", ' + || ' "Item"."invoicePaidAt" at time zone ''UTC'' AS "invoicePaidAtUTC", ' + || ' to_jsonb(users.*) as user, ' + || ' g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore" ' + || ' FROM "Item" ' + || ' JOIN users ON users.id = "Item"."userId" ' + || ' LEFT JOIN hot_score_view g ON g.id = "Item".id ' + || ' WHERE "Item".path <@ (SELECT path FROM "Item" WHERE id = $1) ' || _where + USING _item_id, _level, _where, _order_by; + + EXECUTE '' + || 'SELECT COALESCE(jsonb_agg(sub), ''[]''::jsonb) AS comments ' + || 'FROM ( ' + || ' SELECT "Item".*, item_comments("Item".id, $2 - 1, $3, $4) AS comments ' + || ' FROM t_item "Item"' + || ' WHERE "Item"."parentId" = $1 ' + || _order_by + || ' ) sub' + INTO result USING _item_id, _level, _where, _order_by; + RETURN result; +END +$$; + +-- add limit and offset +CREATE OR REPLACE FUNCTION item_comments_limited( + _item_id int, _limit int, _offset int, _grandchild_limit int, + _level int, _where text, _order_by text) + RETURNS jsonb + LANGUAGE plpgsql VOLATILE PARALLEL SAFE AS +$$ +DECLARE + result jsonb; +BEGIN + IF _level < 1 THEN + RETURN '[]'::jsonb; + END IF; + + EXECUTE 'CREATE TEMP TABLE IF NOT EXISTS t_item ON COMMIT DROP AS ' + || 'WITH RECURSIVE base AS ( ' + || ' (SELECT "Item".*, 1 as level, ROW_NUMBER() OVER () as rn ' + || ' FROM "Item" ' + || ' WHERE "Item"."parentId" = $1 ' + || _order_by || ' ' + || ' LIMIT $2 ' + || ' OFFSET $3) ' + || ' UNION ALL ' + || ' (SELECT "Item".*, b.level + 1, ROW_NUMBER() OVER (PARTITION BY "Item"."parentId" ' || _order_by || ') ' + || ' FROM "Item" ' + || ' JOIN base b ON "Item"."parentId" = b.id ' + || ' LEFT JOIN hot_score_view g ON g.id = "Item".id ' + || ' WHERE b.level < $5 AND (b.level = 1 OR b.rn <= $4)) ' + || ') ' + || 'SELECT "Item".*, "Item".created_at at time zone ''UTC'' AS "createdAt", "Item".updated_at at time zone ''UTC'' AS "updatedAt", ' + || ' "Item"."invoicePaidAt" at time zone ''UTC'' AS "invoicePaidAtUTC", ' + || ' to_jsonb(users.*) as user, ' + || ' g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore" ' + || 'FROM base "Item" ' + || 'JOIN users ON users.id = "Item"."userId" ' + || 'LEFT JOIN hot_score_view g ON g.id = "Item".id ' + || 'WHERE ("Item".level = 1 OR "Item".rn <= $4 - "Item".level + 2) ' || _where + USING _item_id, _limit, _offset, _grandchild_limit, _level, _where, _order_by; + + + EXECUTE '' + || 'SELECT COALESCE(jsonb_agg(sub), ''[]''::jsonb) AS comments ' + || 'FROM ( ' + || ' SELECT "Item".*, item_comments_limited("Item".id, $2, $3, $4, $5 - 1, $6, $7) AS comments ' + || ' FROM t_item "Item" ' + || ' WHERE "Item"."parentId" = $1 ' + || _order_by + || ' ) sub' + INTO result USING _item_id, _limit, _offset, _grandchild_limit, _level, _where, _order_by; + RETURN result; +END +$$; + + diff --git a/prisma/schema.prisma b/prisma/schema.prisma index a3f986b6..12533a4f 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -148,6 +148,7 @@ model User { directReceive Boolean @default(true) DirectPaymentReceived DirectPayment[] @relation("DirectPaymentReceived") DirectPaymentSent DirectPayment[] @relation("DirectPaymentSent") + UserSubTrust UserSubTrust[] @@index([photoId]) @@index([createdAt], map: "users.created_at_index") @@ -184,6 +185,21 @@ model OneDayReferral { @@index([type, typeId]) } +model UserSubTrust { + subName String @db.Citext + userId Int + zapPostTrust Float @default(0) + subZapPostTrust Float @default(0) + zapCommentTrust Float @default(0) + subZapCommentTrust Float @default(0) + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + sub Sub @relation(fields: [subName], references: [name], onDelete: Cascade) + + @@id([userId, subName]) +} + enum WalletType { LIGHTNING_ADDRESS LND @@ -498,85 +514,87 @@ model Message { /// This model contains an expression index which requires additional setup for migrations. Visit https://pris.ly/d/expression-indexes for more info. model Item { - id Int @id @default(autoincrement()) - createdAt DateTime @default(now()) @map("created_at") - updatedAt DateTime @default(now()) @updatedAt @map("updated_at") - title String? - text String? - url String? - userId Int - parentId Int? - path Unsupported("ltree")? - pinId Int? - latitude Float? - location String? - longitude Float? - maxBid Int? - maxSalary Int? - minSalary Int? - remote Boolean? - subName String? @db.Citext - statusUpdatedAt DateTime? - status Status @default(ACTIVE) - company String? - weightedVotes Float @default(0) - boost Int @default(0) - oldBoost Int @default(0) - pollCost Int? - paidImgLink Boolean @default(false) - commentMsats BigInt @default(0) - commentMcredits BigInt @default(0) - lastCommentAt DateTime? - lastZapAt DateTime? - ncomments Int @default(0) - nDirectComments Int @default(0) - msats BigInt @default(0) - mcredits BigInt @default(0) - cost Int @default(0) - weightedDownVotes Float @default(0) - bio Boolean @default(false) - freebie Boolean @default(false) - deletedAt DateTime? - otsFile Bytes? - otsHash String? - imgproxyUrls Json? - bounty Int? - noteId String? @unique(map: "Item.noteId_unique") - rootId Int? - bountyPaidTo Int[] - upvotes Int @default(0) - weightedComments Float @default(0) - Bookmark Bookmark[] - parent Item? @relation("ParentChildren", fields: [parentId], references: [id]) - children Item[] @relation("ParentChildren") - pin Pin? @relation(fields: [pinId], references: [id]) - root Item? @relation("RootDescendant", fields: [rootId], references: [id]) - descendants Item[] @relation("RootDescendant") - sub Sub? @relation(fields: [subName], references: [name], onDelete: Cascade, onUpdate: Cascade) - user User @relation("UserItems", fields: [userId], references: [id], onDelete: Cascade) - itemActs ItemAct[] - mentions Mention[] - itemReferrers ItemMention[] @relation("referrer") - itemReferees ItemMention[] @relation("referee") - pollOptions PollOption[] - PollVote PollVote[] - threadSubscriptions ThreadSubscription[] - User User[] - itemForwards ItemForward[] - itemUploads ItemUpload[] - uploadId Int? - invoiceId Int? - invoiceActionState InvoiceActionState? - invoicePaidAt DateTime? - outlawed Boolean @default(false) - apiKey Boolean @default(false) - pollExpiresAt DateTime? - Ancestors Reply[] @relation("AncestorReplyItem") - Replies Reply[] - Reminder Reminder[] - invoice Invoice? @relation(fields: [invoiceId], references: [id], onDelete: SetNull) - PollBlindVote PollBlindVote[] - ItemUserAgg ItemUserAgg[] + id Int @id @default(autoincrement()) + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") + title String? + text String? + url String? + userId Int + parentId Int? + path Unsupported("ltree")? + pinId Int? + latitude Float? + location String? + longitude Float? + maxBid Int? + maxSalary Int? + minSalary Int? + remote Boolean? + subName String? @db.Citext + statusUpdatedAt DateTime? + status Status @default(ACTIVE) + company String? + weightedVotes Float @default(0) + subWeightedVotes Float @default(0) + boost Int @default(0) + oldBoost Int @default(0) + pollCost Int? + paidImgLink Boolean @default(false) + commentMsats BigInt @default(0) + commentMcredits BigInt @default(0) + lastCommentAt DateTime? + lastZapAt DateTime? + ncomments Int @default(0) + nDirectComments Int @default(0) + msats BigInt @default(0) + mcredits BigInt @default(0) + cost Int @default(0) + weightedDownVotes Float @default(0) + subWeightedDownVotes Float @default(0) + bio Boolean @default(false) + freebie Boolean @default(false) + deletedAt DateTime? + otsFile Bytes? + otsHash String? + imgproxyUrls Json? + bounty Int? + noteId String? @unique(map: "Item.noteId_unique") + rootId Int? + bountyPaidTo Int[] + upvotes Int @default(0) + weightedComments Float @default(0) + Bookmark Bookmark[] + parent Item? @relation("ParentChildren", fields: [parentId], references: [id]) + children Item[] @relation("ParentChildren") + pin Pin? @relation(fields: [pinId], references: [id]) + root Item? @relation("RootDescendant", fields: [rootId], references: [id]) + descendants Item[] @relation("RootDescendant") + sub Sub? @relation(fields: [subName], references: [name], onDelete: Cascade, onUpdate: Cascade) + user User @relation("UserItems", fields: [userId], references: [id], onDelete: Cascade) + itemActs ItemAct[] + mentions Mention[] + itemReferrers ItemMention[] @relation("referrer") + itemReferees ItemMention[] @relation("referee") + pollOptions PollOption[] + PollVote PollVote[] + threadSubscriptions ThreadSubscription[] + User User[] + itemForwards ItemForward[] + itemUploads ItemUpload[] + uploadId Int? + invoiceId Int? + invoiceActionState InvoiceActionState? + invoicePaidAt DateTime? + outlawed Boolean @default(false) + apiKey Boolean @default(false) + pollExpiresAt DateTime? + Ancestors Reply[] @relation("AncestorReplyItem") + Replies Reply[] + Reminder Reminder[] + invoice Invoice? @relation(fields: [invoiceId], references: [id], onDelete: SetNull) + PollBlindVote PollBlindVote[] + ItemUserAgg ItemUserAgg[] @@index([uploadId]) @@index([lastZapAt]) @@ -760,6 +778,7 @@ model Sub { MuteSub MuteSub[] SubSubscription SubSubscription[] TerritoryTransfer TerritoryTransfer[] + UserSubTrust UserSubTrust[] @@index([parentName]) @@index([createdAt]) diff --git a/worker/trust.js b/worker/trust.js index 8c025e31..b861aa86 100644 --- a/worker/trust.js +++ b/worker/trust.js @@ -1,38 +1,89 @@ import * as math from 'mathjs' -import { USER_ID, SN_ADMIN_IDS } from '@/lib/constants' +import { USER_ID } from '@/lib/constants' +import { Prisma } from '@prisma/client' +import { initialTrust, GLOBAL_SEEDS } from '@/api/paidAction/lib/territory' -export async function trust ({ boss, models }) { - try { - console.time('trust') - console.timeLog('trust', 'getting graph') - const graph = await getGraph(models) - console.timeLog('trust', 'computing trust') - const [vGlobal, mPersonal] = await trustGivenGraph(graph) - console.timeLog('trust', 'storing trust') - await storeTrust(models, graph, vGlobal, mPersonal) - } finally { - console.timeEnd('trust') - } -} - -const MAX_DEPTH = 10 +const MAX_DEPTH = 40 const MAX_TRUST = 1 -const MIN_SUCCESS = 1 +const MIN_SUCCESS = 0 // https://en.wikipedia.org/wiki/Normal_distribution#Quantile_function const Z_CONFIDENCE = 6.109410204869 // 99.9999999% confidence -const GLOBAL_ROOT = 616 -const SEED_WEIGHT = 1.0 +const SEED_WEIGHT = 0.83 const AGAINST_MSAT_MIN = 1000 -const MSAT_MIN = 20001 // 20001 is the minimum for a tip to be counted in trust -const SIG_DIFF = 0.1 // need to differ by at least 10 percent +const MSAT_MIN = 1001 // 20001 is the minimum for a tip to be counted in trust +const INDEPENDENCE_THRESHOLD = 50 // how many zappers are needed to consider a sub independent +const IRRELEVANT_CUMULATIVE_TRUST = 0.001 // if a user has less than this amount of cumulative trust, they are irrelevant + +// for each subName, we'll need to get two graphs +// one for comments and one for posts +// then we'll need to do two trust calculations on each graph +// one with global seeds and one with subName seeds +export async function trust ({ boss, models }) { + console.time('trust') + const territories = await models.sub.findMany({ + where: { + status: 'ACTIVE' + } + }) + for (const territory of territories) { + const seeds = GLOBAL_SEEDS.includes(territory.userId) ? GLOBAL_SEEDS : GLOBAL_SEEDS.concat(territory.userId) + try { + console.timeLog('trust', `getting post graph for ${territory.name}`) + const postGraph = await getGraph(models, territory.name, true, seeds) + console.timeLog('trust', `getting comment graph for ${territory.name}`) + const commentGraph = await getGraph(models, territory.name, false, seeds) + console.timeLog('trust', `computing global post trust for ${territory.name}`) + const vGlobalPost = await trustGivenGraph(postGraph) + console.timeLog('trust', `computing global comment trust for ${territory.name}`) + const vGlobalComment = await trustGivenGraph(commentGraph) + console.timeLog('trust', `computing sub post trust for ${territory.name}`) + const vSubPost = await trustGivenGraph(postGraph, postGraph.length > INDEPENDENCE_THRESHOLD ? [territory.userId] : seeds) + console.timeLog('trust', `computing sub comment trust for ${territory.name}`) + const vSubComment = await trustGivenGraph(commentGraph, commentGraph.length > INDEPENDENCE_THRESHOLD ? [territory.userId] : seeds) + console.timeLog('trust', `storing trust for ${territory.name}`) + let results = reduceVectors(territory.name, { + zapPostTrust: { + graph: postGraph, + vector: vGlobalPost + }, + subZapPostTrust: { + graph: postGraph, + vector: vSubPost + }, + zapCommentTrust: { + graph: commentGraph, + vector: vGlobalComment + }, + subZapCommentTrust: { + graph: commentGraph, + vector: vSubComment + } + }) + + if (results.length === 0) { + console.timeLog('trust', `no results for ${territory.name} - adding seeds`) + results = initialTrust({ name: territory.name, userId: territory.userId }) + } + + await storeTrust(models, territory.name, results) + } catch (e) { + console.error(`error computing trust for ${territory.name}:`, e) + } finally { + console.timeLog('trust', `finished computing trust for ${territory.name}`) + } + } + console.timeEnd('trust') +} /* Given a graph and start this function returns an object where the keys are the node id and their value is the trust of that node */ -function trustGivenGraph (graph) { +// I'm going to need to send subName, and multiply by a vector instead of a matrix +function trustGivenGraph (graph, seeds = GLOBAL_SEEDS) { + console.timeLog('trust', `creating matrix of size ${graph.length} x ${graph.length}`) // empty matrix of proper size nstackers x nstackers - let mat = math.zeros(graph.length, graph.length, 'sparse') + const mat = math.zeros(graph.length, graph.length, 'sparse') // create a map of user id to position in matrix const posByUserId = {} @@ -54,54 +105,57 @@ function trustGivenGraph (graph) { // perform random walk over trust matrix // the resulting matrix columns represent the trust a user (col) has for each other user (rows) - // XXX this scales N^3 and mathjs is slow - let matT = math.transpose(mat) - const original = matT.clone() - for (let i = 0; i < MAX_DEPTH; i++) { - console.timeLog('trust', `matrix multiply ${i}`) - matT = math.multiply(original, matT) - matT = math.add(math.multiply(1 - SEED_WEIGHT, matT), math.multiply(SEED_WEIGHT, original)) + const matT = math.transpose(mat) + const vTrust = math.zeros(graph.length) + for (const seed of seeds) { + vTrust.set([posByUserId[seed], 0], 1.0 / seeds.length) } + let result = vTrust.clone() + console.timeLog('trust', 'matrix multiply') + for (let i = 0; i < MAX_DEPTH; i++) { + result = math.multiply(matT, result) + result = math.add(math.multiply(1 - SEED_WEIGHT, result), math.multiply(SEED_WEIGHT, vTrust)) + } + result = math.squeeze(result) console.timeLog('trust', 'transforming result') - const seedIdxs = SN_ADMIN_IDS.map(id => posByUserId[id]) - const isOutlier = (fromIdx, idx) => [...seedIdxs, fromIdx].includes(idx) - const sqapply = (mat, fn) => { - let idx = 0 - return math.squeeze(math.apply(mat, 1, d => { - const filtered = math.filter(d, (val, fidx) => { - return val !== 0 && !isOutlier(idx, fidx[0]) - }) - idx++ - if (filtered.length === 0) return 0 - return fn(filtered) - })) + const seedIdxs = seeds.map(id => posByUserId[id]) + const filterZeroAndSeed = (val, idx) => { + return val !== 0 && !seedIdxs.includes(idx[0]) + } + const filterSeed = (val, idx) => { + return !seedIdxs.includes(idx[0]) + } + const sqapply = (vec, filterFn, fn) => { + // if the vector is smaller than the seeds, don't filter + const filtered = vec.size()[0] > seeds.length ? math.filter(vec, filterFn) : vec + if (filtered.size()[0] === 0) return 0 + return fn(filtered) } console.timeLog('trust', 'normalizing') console.timeLog('trust', 'stats') - mat = math.transpose(matT) - const std = sqapply(mat, math.std) // math.squeeze(math.std(mat, 1)) - const mean = sqapply(mat, math.mean) // math.squeeze(math.mean(mat, 1)) - const zscore = math.map(mat, (val, idx) => { - const zstd = math.subset(std, math.index(idx[0], 0)) - const zmean = math.subset(mean, math.index(idx[0], 0)) - return zstd ? (val - zmean) / zstd : 0 + const std = sqapply(result, filterZeroAndSeed, math.std) // math.squeeze(math.std(mat, 1)) + const mean = sqapply(result, filterZeroAndSeed, math.mean) // math.squeeze(math.mean(mat, 1)) + console.timeLog('trust', 'std', std) + console.timeLog('trust', 'mean', mean) + const zscore = math.map(result, (val) => { + if (std === 0) return 0 + return (val - mean) / std }) console.timeLog('trust', 'minmax') - const min = sqapply(zscore, math.min) // math.squeeze(math.min(zscore, 1)) - const max = sqapply(zscore, math.max) // math.squeeze(math.max(zscore, 1)) - const mPersonal = math.map(zscore, (val, idx) => { - const zmin = math.subset(min, math.index(idx[0], 0)) - const zmax = math.subset(max, math.index(idx[0], 0)) - const zrange = zmax - zmin - if (val > zmax) return MAX_TRUST - return zrange ? (val - zmin) / zrange : 0 + const min = sqapply(zscore, filterSeed, math.min) // math.squeeze(math.min(zscore, 1)) + const max = sqapply(zscore, filterSeed, math.max) // math.squeeze(math.max(zscore, 1)) + console.timeLog('trust', 'min', min) + console.timeLog('trust', 'max', max) + const normalized = math.map(zscore, (val) => { + const zrange = max - min + if (val > max) return MAX_TRUST + return zrange ? (val - min) / zrange : 0 }) - const vGlobal = math.squeeze(math.row(mPersonal, posByUserId[GLOBAL_ROOT])) - return [vGlobal, mPersonal] + return normalized } /* @@ -111,7 +165,9 @@ function trustGivenGraph (graph) { ... ] */ -async function getGraph (models) { +// I'm going to want to send subName to this function +// and whether it's for comments or posts +async function getGraph (models, subName, postTrust = true, seeds = GLOBAL_SEEDS) { return await models.$queryRaw` SELECT id, json_agg(json_build_object( 'node', oid, @@ -124,10 +180,16 @@ async function getGraph (models) { sum("ItemAct".msats) as user_msats FROM "ItemAct" JOIN "Item" ON "Item".id = "ItemAct"."itemId" AND "ItemAct".act IN ('FEE', 'TIP', 'DONT_LIKE_THIS') - AND "Item"."parentId" IS NULL AND NOT "Item".bio AND "Item"."userId" <> "ItemAct"."userId" + AND NOT "Item".bio AND "Item"."userId" <> "ItemAct"."userId" + AND ${postTrust + ? Prisma.sql`"Item"."parentId" IS NULL AND "Item"."subName" = ${subName}::TEXT` + : Prisma.sql` + "Item"."parentId" IS NOT NULL + JOIN "Item" root ON "Item"."rootId" = root.id AND root."subName" = ${subName}::TEXT` + } JOIN users ON "ItemAct"."userId" = users.id AND users.id <> ${USER_ID.anon} - WHERE "ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID' - GROUP BY user_id, name, item_id, user_at, against + WHERE ("ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID') + GROUP BY user_id, users.name, item_id, user_at, against HAVING CASE WHEN "ItemAct".act = 'DONT_LIKE_THIS' THEN sum("ItemAct".msats) > ${AGAINST_MSAT_MIN} ELSE sum("ItemAct".msats) > ${MSAT_MIN} END @@ -136,7 +198,7 @@ async function getGraph (models) { SELECT a.user_id AS a_id, b.user_id AS b_id, sum(CASE WHEN b.user_msats > a.user_msats THEN a.user_msats / b.user_msats::FLOAT ELSE b.user_msats / a.user_msats::FLOAT END) FILTER(WHERE a.act_at > b.act_at AND a.against = b.against) AS before, sum(CASE WHEN b.user_msats > a.user_msats THEN a.user_msats / b.user_msats::FLOAT ELSE b.user_msats / a.user_msats::FLOAT END) FILTER(WHERE b.act_at > a.act_at AND a.against = b.against) AS after, - sum(log(1 + a.user_msats / 10000::float) + log(1 + b.user_msats / 10000::float)) FILTER(WHERE a.against <> b.against) AS disagree, + count(*) FILTER(WHERE a.against <> b.against) AS disagree, b.user_vote_count AS b_total, a.user_vote_count AS a_total FROM user_votes a JOIN user_votes b ON a.item_id = b.item_id @@ -149,14 +211,9 @@ async function getGraph (models) { confidence(before - disagree, b_total - after, ${Z_CONFIDENCE}) ELSE 0 END AS trust FROM user_pair - WHERE NOT (b_id = ANY (${SN_ADMIN_IDS})) UNION ALL - SELECT a_id AS id, seed_id AS oid, ${MAX_TRUST}::numeric as trust - FROM user_pair, unnest(${SN_ADMIN_IDS}::int[]) seed_id - GROUP BY a_id, a_total, seed_id - UNION ALL - SELECT a_id AS id, a_id AS oid, ${MAX_TRUST}::float as trust - FROM user_pair + SELECT seed_id AS id, seed_id AS oid, 0 AS trust + FROM unnest(${seeds}::int[]) seed_id ) SELECT id, oid, trust, sum(trust) OVER (PARTITION BY id) AS total_trust FROM trust_pairs @@ -165,46 +222,45 @@ async function getGraph (models) { ORDER BY id ASC` } -async function storeTrust (models, graph, vGlobal, mPersonal) { - // convert nodeTrust into table literal string - let globalValues = '' - let personalValues = '' - vGlobal.forEach((val, [idx]) => { - if (isNaN(val)) return - if (globalValues) globalValues += ',' - globalValues += `(${graph[idx].id}, ${val}::FLOAT)` - if (personalValues) personalValues += ',' - personalValues += `(${GLOBAL_ROOT}, ${graph[idx].id}, ${val}::FLOAT)` - }) +function reduceVectors (subName, fieldGraphVectors) { + function reduceVector (field, graph, vector, result = {}) { + vector.forEach((val, [idx]) => { + if (isNaN(val) || val <= 0) return + result[graph[idx].id] = { + ...result[graph[idx].id], + subName, + userId: graph[idx].id, + [field]: val + } + }) + return result + } - math.forEach(mPersonal, (val, [fromIdx, toIdx]) => { - const globalVal = vGlobal.get([toIdx, 0]) - if (isNaN(val) || val - globalVal <= SIG_DIFF) return - if (personalValues) personalValues += ',' - personalValues += `(${graph[fromIdx].id}, ${graph[toIdx].id}, ${val}::FLOAT)` - }) + let result = {} + for (const field in fieldGraphVectors) { + result = reduceVector(field, fieldGraphVectors[field].graph, fieldGraphVectors[field].vector, result) + } + // return only the users with trust > 0 + return Object.values(result).filter(s => + Object.keys(fieldGraphVectors).reduce( + (acc, key) => acc + (s[key] ?? 0), + 0 + ) > IRRELEVANT_CUMULATIVE_TRUST + ) +} + +async function storeTrust (models, subName, results) { + console.timeLog('trust', `storing trust for ${subName} with ${results.length} users`) // update the trust of each user in graph await models.$transaction([ - models.$executeRaw`UPDATE users SET trust = 0`, - models.$executeRawUnsafe( - `UPDATE users - SET trust = g.trust - FROM (values ${globalValues}) g(id, trust) - WHERE users.id = g.id`), - models.$executeRawUnsafe( - `INSERT INTO "Arc" ("fromId", "toId", "zapTrust") - SELECT id, oid, trust - FROM (values ${personalValues}) g(id, oid, trust) - ON CONFLICT ("fromId", "toId") DO UPDATE SET "zapTrust" = EXCLUDED."zapTrust"` - ), - // select all arcs that don't exist in personalValues and delete them - models.$executeRawUnsafe( - `DELETE FROM "Arc" - WHERE ("fromId", "toId") NOT IN ( - SELECT id, oid - FROM (values ${personalValues}) g(id, oid, trust) - )` - ) + models.userSubTrust.deleteMany({ + where: { + subName + } + }), + models.userSubTrust.createMany({ + data: results + }) ]) } diff --git a/worker/views.js b/worker/views.js index e358b5f6..3d7ee530 100644 --- a/worker/views.js +++ b/worker/views.js @@ -32,7 +32,7 @@ export async function rankViews () { const models = createPrisma({ connectionParams: { connection_limit: 1 } }) try { - for (const view of ['zap_rank_personal_view']) { + for (const view of ['hot_score_view']) { await models.$queryRawUnsafe(`REFRESH MATERIALIZED VIEW CONCURRENTLY ${view}`) } } finally {