ekzyis/stacker.news
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

territory specific trust (#1965)

Browse Source 
* territory specific trust

* functional parity with master

* revert back to materialized view for ranking

* update query for populating subWeightedVotes

* fix anon hot comments

* fix zap denormalization, change weightedComments to be for zaps, order updates of ancestors to prevent deadlocks

* reduce weight of comment zaps for hot score

* do zap ancestor updates together

* initialize trust in new/unpopular territories

* simplify denormalization of zap/downzaps

* recompute all scores
This commit is contained in:
Keyan 2025-03-15 08:11:33 -05:00 committed by GitHub
parent 53f6c34ee7
commit b672d015e2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 735 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
api/paidAction/downZap.js
View File

@ -1,5 +1,6 @@
import { PAID_ACTION_PAYMENT_METHODS } from '@/lib/constants'
import { msatsToSats, satsToMsats } from '@/lib/format'
import { Prisma } from '@prisma/client'
export const anonable = false
@ -48,9 +49,9 @@ export async function onPaid ({ invoice, actId }, { tx }) {
let itemAct
if (invoice) {
await tx.itemAct.updateMany({ where: { invoiceId: invoice.id }, data: { invoiceActionState: 'PAID' } })
itemAct = await tx.itemAct.findFirst({ where: { invoiceId: invoice.id } })
itemAct = await tx.itemAct.findFirst({ where: { invoiceId: invoice.id }, include: { item: true } })
} else if (actId) {
itemAct = await tx.itemAct.findUnique({ where: { id: actId } })
itemAct = await tx.itemAct.findUnique({ where: { id: actId }, include: { item: true } })
} else {
throw new Error('No invoice or actId')
}
@ -60,19 +61,34 @@ export async function onPaid ({ invoice, actId }, { tx }) {
// denormalize downzaps
await tx.$executeRaw`
WITH zapper AS (
SELECT trust FROM users WHERE id = ${itemAct.userId}::INTEGER
), zap AS (
INSERT INTO "ItemUserAgg" ("userId", "itemId", "downZapSats")
VALUES (${itemAct.userId}::INTEGER, ${itemAct.itemId}::INTEGER, ${sats}::INTEGER)
ON CONFLICT ("itemId", "userId") DO UPDATE
SET "downZapSats" = "ItemUserAgg"."downZapSats" + ${sats}::INTEGER, updated_at = now()
RETURNING LOG("downZapSats" / GREATEST("downZapSats" - ${sats}::INTEGER, 1)::FLOAT) AS log_sats
)
UPDATE "Item"
SET "weightedDownVotes" = "weightedDownVotes" + (zapper.trust * zap.log_sats)
FROM zap, zapper
WHERE "Item".id = ${itemAct.itemId}::INTEGER`
WITH territory AS (
SELECT COALESCE(r."subName", i."subName", 'meta')::TEXT as "subName"
FROM "Item" i
LEFT JOIN "Item" r ON r.id = i."rootId"
WHERE i.id = ${itemAct.itemId}::INTEGER
), zapper AS (
SELECT
COALESCE(${itemAct.item.parentId
? Prisma.sql`"zapCommentTrust"`
: Prisma.sql`"zapPostTrust"`}, 0) as "zapTrust",
COALESCE(${itemAct.item.parentId
? Prisma.sql`"subZapCommentTrust"`
: Prisma.sql`"subZapPostTrust"`}, 0) as "subZapTrust"
FROM territory
LEFT JOIN "UserSubTrust" ust ON ust."subName" = territory."subName"
AND ust."userId" = ${itemAct.userId}::INTEGER
), zap AS (
INSERT INTO "ItemUserAgg" ("userId", "itemId", "downZapSats")
VALUES (${itemAct.userId}::INTEGER, ${itemAct.itemId}::INTEGER, ${sats}::INTEGER)
ON CONFLICT ("itemId", "userId") DO UPDATE
SET "downZapSats" = "ItemUserAgg"."downZapSats" + ${sats}::INTEGER, updated_at = now()
RETURNING LOG("downZapSats" / GREATEST("downZapSats" - ${sats}::INTEGER, 1)::FLOAT) AS log_sats
)
UPDATE "Item"
SET "weightedDownVotes" = "weightedDownVotes" + zapper."zapTrust" * zap.log_sats,
"subWeightedDownVotes" = "subWeightedDownVotes" + zapper."subZapTrust" * zap.log_sats
FROM zap, zapper
WHERE "Item".id = ${itemAct.itemId}::INTEGER`
}
export async function onFail ({ invoice }, { tx }) {

11
api/paidAction/itemCreate.js
View File

@ -252,15 +252,18 @@ export async function onPaid ({ invoice, id }, context) {
JOIN users ON "Item"."userId" = users.id
WHERE "Item".id = ${item.id}::INTEGER
), ancestors AS (
SELECT "Item".*
FROM "Item", comment
WHERE "Item".path @> comment.path AND "Item".id <> comment.id
ORDER BY "Item".id
), updated_ancestors AS (
UPDATE "Item"
SET ncomments = "Item".ncomments + 1,
"lastCommentAt" = GREATEST("Item"."lastCommentAt", comment.created_at),
"weightedComments" = "Item"."weightedComments" +
CASE WHEN comment."userId" = "Item"."userId" THEN 0 ELSE comment.trust END,
"nDirectComments" = "Item"."nDirectComments" +
CASE WHEN comment."parentId" = "Item".id THEN 1 ELSE 0 END
FROM comment
WHERE "Item".path @> comment.path AND "Item".id <> comment.id
FROM comment, ancestors
WHERE "Item".id = ancestors.id
RETURNING "Item".*
)
INSERT INTO "Reply" (created_at, updated_at, "ancestorId", "ancestorUserId", "itemId", "userId", level)

27
api/paidAction/lib/territory.js Normal file
View File

@ -0,0 +1,27 @@
import { USER_ID } from '@/lib/constants'
export const GLOBAL_SEEDS = [USER_ID.k00b, USER_ID.ek]
export function initialTrust ({ name, userId }) {
const results = GLOBAL_SEEDS.map(id => ({
subName: name,
userId: id,
zapPostTrust: 1,
subZapPostTrust: 1,
zapCommentTrust: 1,
subZapCommentTrust: 1
}))
if (!GLOBAL_SEEDS.includes(userId)) {
results.push({
subName: name,
userId,
zapPostTrust: 0,
subZapPostTrust: 1,
zapCommentTrust: 0,
subZapCommentTrust: 1
})
}
return results
}

9
api/paidAction/territoryCreate.js
View File

@ -1,6 +1,7 @@
import { PAID_ACTION_PAYMENT_METHODS, TERRITORY_PERIOD_COST } from '@/lib/constants'
import { satsToMsats } from '@/lib/format'
import { nextBilling } from '@/lib/territory'
import { initialTrust } from './lib/territory'
export const anonable = false
@ -20,7 +21,7 @@ export async function perform ({ invoiceId, ...data }, { me, cost, tx }) {
const billedLastAt = new Date()
const billPaidUntil = nextBilling(billedLastAt, billingType)
return await tx.sub.create({
const sub = await tx.sub.create({
data: {
...data,
billedLastAt,
@ -42,6 +43,12 @@ export async function perform ({ invoiceId, ...data }, { me, cost, tx }) {
}
}
})
await tx.userSubTrust.createMany({
data: initialTrust({ name: sub.name, userId: sub.userId })
})
return sub
}
export async function describe ({ name }) {

9
api/paidAction/territoryUnarchive.js
View File

@ -1,6 +1,7 @@
import { PAID_ACTION_PAYMENT_METHODS, TERRITORY_PERIOD_COST } from '@/lib/constants'
import { satsToMsats } from '@/lib/format'
import { nextBilling } from '@/lib/territory'
import { initialTrust } from './lib/territory'
export const anonable = false
@ -65,7 +66,7 @@ export async function perform ({ name, invoiceId, ...data }, { me, cost, tx }) {
}
})
return await tx.sub.update({
const updatedSub = await tx.sub.update({
data,
// optimistic concurrency control
// make sure none of the relevant fields have changed since we fetched the sub
@ -76,6 +77,12 @@ export async function perform ({ name, invoiceId, ...data }, { me, cost, tx }) {
}
}
})
await tx.userSubTrust.createMany({
data: initialTrust({ name: updatedSub.name, userId: updatedSub.userId })
})
return updatedSub
}
export async function describe ({ name }, context) {

61
api/paidAction/zap.js
View File

@ -2,6 +2,7 @@ import { PAID_ACTION_PAYMENT_METHODS, USER_ID } from '@/lib/constants'
import { msatsToSats, satsToMsats } from '@/lib/format'
import { notifyZapped } from '@/lib/webPush'
import { getInvoiceableWallets } from '@/wallets/server'
import { Prisma } from '@prisma/client'
export const anonable = true
@ -149,8 +150,22 @@ export async function onPaid ({ invoice, actIds }, { tx }) {
// perform denomormalized aggregates: weighted votes, upvotes, msats, lastZapAt
// NOTE: for the rows that might be updated by a concurrent zap, we use UPDATE for implicit locking
await tx.$queryRaw`
WITH zapper AS (
SELECT trust FROM users WHERE id = ${itemAct.userId}::INTEGER
WITH territory AS (
SELECT COALESCE(r."subName", i."subName", 'meta')::TEXT as "subName"
FROM "Item" i
LEFT JOIN "Item" r ON r.id = i."rootId"
WHERE i.id = ${itemAct.itemId}::INTEGER
), zapper AS (
SELECT
COALESCE(${itemAct.item.parentId
? Prisma.sql`"zapCommentTrust"`
: Prisma.sql`"zapPostTrust"`}, 0) as "zapTrust",
COALESCE(${itemAct.item.parentId
? Prisma.sql`"subZapCommentTrust"`
: Prisma.sql`"subZapPostTrust"`}, 0) as "subZapTrust"
FROM territory
LEFT JOIN "UserSubTrust" ust ON ust."subName" = territory."subName"
AND ust."userId" = ${itemAct.userId}::INTEGER
), zap AS (
INSERT INTO "ItemUserAgg" ("userId", "itemId", "zapSats")
VALUES (${itemAct.userId}::INTEGER, ${itemAct.itemId}::INTEGER, ${sats}::INTEGER)
@ -158,17 +173,30 @@ export async function onPaid ({ invoice, actIds }, { tx }) {
SET "zapSats" = "ItemUserAgg"."zapSats" + ${sats}::INTEGER, updated_at = now()
RETURNING ("zapSats" = ${sats}::INTEGER)::INTEGER as first_vote,
LOG("zapSats" / GREATEST("zapSats" - ${sats}::INTEGER, 1)::FLOAT) AS log_sats
), item_zapped AS (
UPDATE "Item"
SET
"weightedVotes" = "weightedVotes" + zapper."zapTrust" * zap.log_sats,
"subWeightedVotes" = "subWeightedVotes" + zapper."subZapTrust" * zap.log_sats,
upvotes = upvotes + zap.first_vote,
msats = "Item".msats + ${msats}::BIGINT,
mcredits = "Item".mcredits + ${invoice?.invoiceForward ? 0n : msats}::BIGINT,
"lastZapAt" = now()
FROM zap, zapper
WHERE "Item".id = ${itemAct.itemId}::INTEGER
RETURNING "Item".*, zapper."zapTrust" * zap.log_sats as "weightedVote"
), ancestors AS (
SELECT "Item".*
FROM "Item", item_zapped
WHERE "Item".path @> item_zapped.path AND "Item".id <> item_zapped.id
ORDER BY "Item".id
)
UPDATE "Item"
SET
"weightedVotes" = "weightedVotes" + (zapper.trust * zap.log_sats),
upvotes = upvotes + zap.first_vote,
msats = "Item".msats + ${msats}::BIGINT,
mcredits = "Item".mcredits + ${invoice?.invoiceForward ? 0n : msats}::BIGINT,
"lastZapAt" = now()
FROM zap, zapper
WHERE "Item".id = ${itemAct.itemId}::INTEGER
RETURNING "Item".*`
SET "weightedComments" = "Item"."weightedComments" + item_zapped."weightedVote",
"commentMsats" = "Item"."commentMsats" + ${msats}::BIGINT,
"commentMcredits" = "Item"."commentMcredits" + ${invoice?.invoiceForward ? 0n : msats}::BIGINT
FROM item_zapped, ancestors
WHERE "Item".id = ancestors.id`
// record potential bounty payment
// NOTE: we are at least guaranteed that we see the update "ItemUserAgg" from our tx so we can trust
@ -188,17 +216,6 @@ export async function onPaid ({ invoice, actIds }, { tx }) {
SET "bountyPaidTo" = array_remove(array_append(array_remove("bountyPaidTo", bounty.target), bounty.target), NULL)
FROM bounty
WHERE "Item".id = bounty.id AND bounty.paid`
// update commentMsats on ancestors
await tx.$executeRaw`
WITH zapped AS (
SELECT * FROM "Item" WHERE id = ${itemAct.itemId}::INTEGER
)
UPDATE "Item"
SET "commentMsats" = "Item"."commentMsats" + ${msats}::BIGINT,
"commentMcredits" = "Item"."commentMcredits" + ${invoice?.invoiceForward ? 0n : msats}::BIGINT
FROM zapped
WHERE "Item".path @> zapped.path AND "Item".id <> zapped.id`
}
export async function nonCriticalSideEffects ({ invoice, actIds }, { models }) {

82
api/resolvers/item.js
View File

@ -39,16 +39,12 @@ function commentsOrderByClause (me, models, sort) {
COALESCE("Item"."invoicePaidAt", "Item".created_at) DESC, "Item".id DESC`
}
if (me && sort === 'hot') {
if (sort === 'hot') {
return `ORDER BY ${sharedSorts},
"personal_hot_score" DESC NULLS LAST,
"Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC`
"hotScore" DESC NULLS LAST,
"Item".msats DESC, "Item".id DESC`
} else {
if (sort === 'top') {
return `ORDER BY ${sharedSorts}, ${orderByNumerator({ models, commentScaler: 0 })} DESC NULLS LAST, "Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC`
} else {
return `ORDER BY ${sharedSorts}, ${orderByNumerator({ models, commentScaler: 0, considerBoost: true })}/POWER(GREATEST(3, EXTRACT(EPOCH FROM (now_utc() - "Item".created_at))/3600), 1.3) DESC NULLS LAST, "Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC`
}
return `ORDER BY ${sharedSorts}, "Item"."weightedVotes" - "Item"."weightedDownVotes" DESC NULLS LAST, "Item".msats DESC, "Item".id DESC`
}
}
@ -138,14 +134,14 @@ export async function getAd (parent, { sub, subArr = [], showNsfw = false }, { m
}, ...subArr))?.[0] || null
}
const orderByClause = (by, me, models, type) => {
const orderByClause = (by, me, models, type, sub) => {
switch (by) {
case 'comments':
return 'ORDER BY "Item".ncomments DESC'
case 'sats':
return 'ORDER BY "Item".msats DESC'
case 'zaprank':
return topOrderByWeightedSats(me, models)
return topOrderByWeightedSats(me, models, sub)
case 'boost':
return 'ORDER BY "Item".boost DESC'
case 'random':
@ -155,22 +151,8 @@ const orderByClause = (by, me, models, type) => {
}
}
export function orderByNumerator ({ models, commentScaler = 0.5, considerBoost = false }) {
return `((CASE WHEN "Item"."weightedVotes" - "Item"."weightedDownVotes" > 0 THEN
GREATEST("Item"."weightedVotes" - "Item"."weightedDownVotes", POWER("Item"."weightedVotes" - "Item"."weightedDownVotes", 1.2))
ELSE
"Item"."weightedVotes" - "Item"."weightedDownVotes"
END + "Item"."weightedComments"*${commentScaler}) + ${considerBoost ? `("Item".boost / ${BOOST_MULT})` : 0})`
}
export function joinZapRankPersonalView (me, models) {
let join = ` JOIN zap_rank_personal_view g ON g.id = "Item".id AND g."viewerId" = ${GLOBAL_SEED} `
if (me) {
join += ` LEFT JOIN zap_rank_personal_view l ON l.id = g.id AND l."viewerId" = ${me.id} `
}
return join
export function joinHotScoreView (me, models) {
return ' JOIN hot_score_view g ON g.id = "Item".id '
}
// this grabs all the stuff we need to display the item list and only
@ -475,10 +457,10 @@ export default {
await filterClause(me, models, type),
by === 'boost' && '"Item".boost > 0',
muteClause(me))}
${orderByClause(by || 'zaprank', me, models, type)}
${orderByClause(by || 'zaprank', me, models, type, sub)}
OFFSET $3
LIMIT $4`,
orderBy: orderByClause(by || 'zaprank', me, models, type)
orderBy: orderByClause(by || 'zaprank', me, models, type, sub)
}, ...whenRange(when, from, to || decodedCursor.time), decodedCursor.offset, limit, ...subArr)
break
case 'random':
@ -571,10 +553,10 @@ export default {
me,
models,
query: `
${SELECT}, ${me ? 'GREATEST(g.tf_hot_score, l.tf_hot_score)' : 'g.tf_hot_score'} AS rank
${SELECT}, g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore"
FROM "Item"
LEFT JOIN "Sub" ON "Sub"."name" = "Item"."subName"
${joinZapRankPersonalView(me, models)}
${joinHotScoreView(me, models)}
${whereClause(
// in home (sub undefined), filter out global pinned items since we inject them later
sub ? '"Item"."pinId" IS NULL' : 'NOT ("Item"."pinId" IS NOT NULL AND "Item"."subName" IS NULL)',
@ -587,40 +569,11 @@ export default {
await filterClause(me, models, type),
subClause(sub, 3, 'Item', me, showNsfw),
muteClause(me))}
ORDER BY rank DESC
ORDER BY ${sub ? '"subHotScore"' : '"hotScore"'} DESC, "Item".msats DESC, "Item".id DESC
OFFSET $1
LIMIT $2`,
orderBy: 'ORDER BY rank DESC'
orderBy: `ORDER BY ${sub ? '"subHotScore"' : '"hotScore"'} DESC, "Item".msats DESC, "Item".id DESC`
}, decodedCursor.offset, limit, ...subArr)
// XXX this is mostly for subs that are really empty
if (items.length < limit) {
items = await itemQueryWithMeta({
me,
models,
query: `
${SELECT}
FROM "Item"
LEFT JOIN "Sub" ON "Sub"."name" = "Item"."subName"
${whereClause(
subClause(sub, 3, 'Item', me, showNsfw),
muteClause(me),
// in home (sub undefined), filter out global pinned items since we inject them later
sub ? '"Item"."pinId" IS NULL' : 'NOT ("Item"."pinId" IS NOT NULL AND "Item"."subName" IS NULL)',
'"Item"."deletedAt" IS NULL',
'"Item"."parentId" IS NULL',
'"Item".bio = false',
ad ? `"Item".id <> ${ad.id}` : '',
activeOrMine(me),
await filterClause(me, models, type))}
ORDER BY ${orderByNumerator({ models, considerBoost: true })}/POWER(GREATEST(3, EXTRACT(EPOCH FROM (now_utc() - "Item".created_at))/3600), 1.3) DESC NULLS LAST,
"Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC
OFFSET $1
LIMIT $2`,
orderBy: `ORDER BY ${orderByNumerator({ models, considerBoost: true })}/POWER(GREATEST(3, EXTRACT(EPOCH FROM (now_utc() - "Item".created_at))/3600), 1.3) DESC NULLS LAST,
"Item".msats DESC, ("Item".cost > 0) DESC, "Item".id DESC`
}, decodedCursor.offset, limit, ...subArr)
}
break
}
break
@ -1574,6 +1527,9 @@ export const SELECT =
`SELECT "Item".*, "Item".created_at as "createdAt", "Item".updated_at as "updatedAt",
ltree2text("Item"."path") AS "path"`
function topOrderByWeightedSats (me, models) {
return `ORDER BY ${orderByNumerator({ models })} DESC NULLS LAST, "Item".id DESC`
function topOrderByWeightedSats (me, models, sub) {
if (sub) {
return 'ORDER BY "Item"."subWeightedVotes" - "Item"."subWeightedDownVotes" DESC, "Item".msats DESC, "Item".id DESC'
}
return 'ORDER BY "Item"."weightedVotes" - "Item"."weightedDownVotes" DESC, "Item".msats DESC, "Item".id DESC'
}

109
prisma/migrations/20250308234113_user_sub_trust/migration.sql Normal file
View File

@ -0,0 +1,109 @@
-- AlterTable
ALTER TABLE "Item" ADD COLUMN "subWeightedDownVotes" FLOAT NOT NULL DEFAULT 0,
ADD COLUMN "subWeightedVotes" FLOAT NOT NULL DEFAULT 0;
CREATE INDEX "Item.sumSubVotes_index" ON "Item"(("subWeightedVotes" - "subWeightedDownVotes"));
-- CreateTable
CREATE TABLE "UserSubTrust" (
"subName" CITEXT NOT NULL,
"userId" INTEGER NOT NULL,
"zapPostTrust" DOUBLE PRECISION NOT NULL DEFAULT 0,
"subZapPostTrust" DOUBLE PRECISION NOT NULL DEFAULT 0,
"zapCommentTrust" DOUBLE PRECISION NOT NULL DEFAULT 0,
"subZapCommentTrust" DOUBLE PRECISION NOT NULL DEFAULT 0,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "UserSubTrust_pkey" PRIMARY KEY ("userId","subName")
);
-- AddForeignKey
ALTER TABLE "UserSubTrust" ADD CONSTRAINT "UserSubTrust_userId_fkey" FOREIGN KEY ("userId") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "UserSubTrust" ADD CONSTRAINT "UserSubTrust_subName_fkey" FOREIGN KEY ("subName") REFERENCES "Sub"("name") ON DELETE CASCADE ON UPDATE CASCADE;
-- UserSubTrust is NOT populated, so this is a no-op ... but useful having it written out for migrating manually on deployment
UPDATE "Item"
SET "subWeightedVotes" = subquery."subWeightedVotes",
"subWeightedDownVotes" = subquery."subWeightedDownVotes",
"weightedVotes" = subquery."weightedVotes",
"weightedDownVotes" = subquery."weightedDownVotes"
FROM (
WITH sub_votes AS (
SELECT "ItemAct"."itemId",
CASE WHEN (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('TIP', 'FEE')))::FLOAT > 0
THEN COALESCE(
LOG(
(SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('TIP', 'FEE')))::FLOAT / 1000
) * CASE
WHEN "Item"."parentId" IS NULL
THEN "UserSubTrust"."subZapPostTrust"
ELSE "UserSubTrust"."subZapCommentTrust"
END, 0)
ELSE 0
END AS "subWeightedVotes",
CASE WHEN (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('TIP', 'FEE')))::FLOAT > 0
THEN COALESCE(
LOG(
(SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('TIP', 'FEE')))::FLOAT / 1000
) * CASE
WHEN "Item"."parentId" IS NULL
THEN "UserSubTrust"."zapPostTrust"
ELSE "UserSubTrust"."zapCommentTrust"
END, 0)
ELSE 0
END AS "weightedVotes",
CASE WHEN (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('DONT_LIKE_THIS')))::FLOAT > 0
THEN COALESCE(
LOG(
(SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('DONT_LIKE_THIS')))::FLOAT / 1000
) * CASE
WHEN "Item"."parentId" IS NULL
THEN "UserSubTrust"."subZapPostTrust"
ELSE "UserSubTrust"."subZapCommentTrust"
END, 0)
ELSE 0
END AS "subWeightedDownVotes",
CASE WHEN (SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('DONT_LIKE_THIS')))::FLOAT > 0
THEN COALESCE(
LOG(
(SUM("ItemAct"."msats") FILTER (WHERE "ItemAct"."act" IN ('DONT_LIKE_THIS')))::FLOAT / 1000
) * CASE
WHEN "Item"."parentId" IS NULL
THEN "UserSubTrust"."zapPostTrust"
ELSE "UserSubTrust"."zapCommentTrust"
END, 0)
ELSE 0
END AS "weightedDownVotes"
FROM "ItemAct"
JOIN "UserSubTrust" ON "ItemAct"."userId" = "UserSubTrust"."userId"
JOIN "Item" ON "Item".id = "ItemAct"."itemId"
AND "UserSubTrust"."subName" = "Item"."subName"
AND "Item"."userId" <> "ItemAct"."userId"
WHERE "ItemAct".act IN ('TIP', 'FEE', 'DONT_LIKE_THIS')
GROUP BY "ItemAct"."itemId", "ItemAct"."userId", "UserSubTrust"."subZapPostTrust",
"UserSubTrust"."subZapCommentTrust", "UserSubTrust"."zapPostTrust", "UserSubTrust"."zapCommentTrust",
"Item"."parentId"
)
SELECT "itemId", SUM("subWeightedVotes") AS "subWeightedVotes", SUM("subWeightedDownVotes") AS "subWeightedDownVotes",
SUM("weightedVotes") AS "weightedVotes", SUM("weightedDownVotes") AS "weightedDownVotes"
FROM sub_votes
GROUP BY "itemId"
) AS subquery
WHERE "Item".id = subquery."itemId";
CREATE MATERIALIZED VIEW IF NOT EXISTS hot_score_view AS
SELECT id,
("Item"."weightedVotes" - "Item"."weightedDownVotes" + ("Item"."weightedComments"*0.25) + ("Item".boost / 5000))
/ POWER(GREATEST(3, EXTRACT(EPOCH FROM (now() - "Item".created_at))/3600), 1.1) AS hot_score,
("Item"."subWeightedVotes" - "Item"."subWeightedDownVotes" + ("Item"."weightedComments"*0.25) + ("Item".boost / 5000))
/ POWER(GREATEST(3, EXTRACT(EPOCH FROM (now() - "Item".created_at))/3600), 1.1) AS sub_hot_score
FROM "Item"
WHERE "Item"."weightedVotes" > 0 OR "Item"."weightedDownVotes" > 0 OR "Item"."subWeightedVotes" > 0
OR "Item"."subWeightedDownVotes" > 0 OR "Item"."weightedComments" > 0 OR "Item".boost > 0;
CREATE UNIQUE INDEX IF NOT EXISTS hot_score_view_id_idx ON hot_score_view(id);
CREATE INDEX IF NOT EXISTS hot_score_view_hot_score_idx ON hot_score_view(hot_score DESC NULLS LAST);
CREATE INDEX IF NOT EXISTS hot_score_view_sub_hot_score_idx ON hot_score_view(sub_hot_score DESC NULLS LAST);

224
prisma/migrations/20250311210455_comments_ranking_changes/migration.sql Normal file
View File

@ -0,0 +1,224 @@
-- add limit and offset
CREATE OR REPLACE FUNCTION item_comments_zaprank_with_me_limited(
_item_id int, _global_seed int, _me_id int, _limit int, _offset int, _grandchild_limit int,
_level int, _where text, _order_by text)
RETURNS jsonb
LANGUAGE plpgsql VOLATILE PARALLEL SAFE AS
$$
DECLARE
result jsonb;
BEGIN
IF _level < 1 THEN
RETURN '[]'::jsonb;
END IF;
EXECUTE 'CREATE TEMP TABLE IF NOT EXISTS t_item ON COMMIT DROP AS '
|| 'WITH RECURSIVE base AS ( '
|| ' (SELECT "Item".*, 1 as level, ROW_NUMBER() OVER () as rn '
|| ' FROM "Item" '
|| ' WHERE "Item"."parentId" = $1 '
|| _order_by || ' '
|| ' LIMIT $4 '
|| ' OFFSET $5) '
|| ' UNION ALL '
|| ' (SELECT "Item".*, b.level + 1, ROW_NUMBER() OVER (PARTITION BY "Item"."parentId" ' || _order_by || ') as rn '
|| ' FROM "Item" '
|| ' JOIN base b ON "Item"."parentId" = b.id '
|| ' LEFT JOIN hot_score_view g ON g.id = "Item".id '
|| ' WHERE b.level < $7 AND (b.level = 1 OR b.rn <= $6)) '
|| ') '
|| 'SELECT "Item".*, '
|| ' "Item".created_at at time zone ''UTC'' AS "createdAt", '
|| ' "Item".updated_at at time zone ''UTC'' AS "updatedAt", '
|| ' "Item"."invoicePaidAt" at time zone ''UTC'' AS "invoicePaidAtUTC", '
|| ' to_jsonb(users.*) || jsonb_build_object(''meMute'', "Mute"."mutedId" IS NOT NULL) AS user, '
|| ' COALESCE("ItemAct"."meMsats", 0) AS "meMsats", '
|| ' COALESCE("ItemAct"."mePendingMsats", 0) as "mePendingMsats", '
|| ' COALESCE("ItemAct"."meDontLikeMsats", 0) AS "meDontLikeMsats", '
|| ' COALESCE("ItemAct"."meMcredits", 0) AS "meMcredits", '
|| ' COALESCE("ItemAct"."mePendingMcredits", 0) as "mePendingMcredits", '
|| ' "Bookmark"."itemId" IS NOT NULL AS "meBookmark", '
|| ' "ThreadSubscription"."itemId" IS NOT NULL AS "meSubscription", '
|| ' g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore" '
|| 'FROM base "Item" '
|| 'JOIN users ON users.id = "Item"."userId" '
|| ' LEFT JOIN "Mute" ON "Mute"."muterId" = $3 AND "Mute"."mutedId" = "Item"."userId" '
|| ' LEFT JOIN "Bookmark" ON "Bookmark"."userId" = $3 AND "Bookmark"."itemId" = "Item".id '
|| ' LEFT JOIN "ThreadSubscription" ON "ThreadSubscription"."userId" = $3 AND "ThreadSubscription"."itemId" = "Item".id '
|| ' LEFT JOIN hot_score_view g ON g.id = "Item".id '
|| 'LEFT JOIN LATERAL ( '
|| ' SELECT "itemId", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND "InvoiceForward".id IS NOT NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "meMsats", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND "InvoiceForward".id IS NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "meMcredits", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS NOT DISTINCT FROM ''PENDING'' AND "InvoiceForward".id IS NOT NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "mePendingMsats", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS NOT DISTINCT FROM ''PENDING'' AND "InvoiceForward".id IS NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "mePendingMcredits", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND act = ''DONT_LIKE_THIS'') AS "meDontLikeMsats" '
|| ' FROM "ItemAct" '
|| ' LEFT JOIN "Invoice" ON "Invoice".id = "ItemAct"."invoiceId" '
|| ' LEFT JOIN "InvoiceForward" ON "InvoiceForward"."invoiceId" = "Invoice"."id" '
|| ' WHERE "ItemAct"."userId" = $3 '
|| ' AND "ItemAct"."itemId" = "Item".id '
|| ' GROUP BY "ItemAct"."itemId" '
|| ') "ItemAct" ON true '
|| 'WHERE ("Item".level = 1 OR "Item".rn <= $6 - "Item".level + 2) ' || _where || ' '
USING _item_id, _global_seed, _me_id, _limit, _offset, _grandchild_limit, _level, _where, _order_by;
EXECUTE ''
|| 'SELECT COALESCE(jsonb_agg(sub), ''[]''::jsonb) AS comments '
|| 'FROM ( '
|| ' SELECT "Item".*, item_comments_zaprank_with_me_limited("Item".id, $2, $3, $4, $5, $6, $7 - 1, $8, $9) AS comments '
|| ' FROM t_item "Item" '
|| ' WHERE "Item"."parentId" = $1 '
|| _order_by
|| ' ) sub'
INTO result USING _item_id, _global_seed, _me_id, _limit, _offset, _grandchild_limit, _level, _where, _order_by;
RETURN result;
END
$$;
-- add cowboy credits
CREATE OR REPLACE FUNCTION item_comments_zaprank_with_me(_item_id int, _global_seed int, _me_id int, _level int, _where text, _order_by text)
RETURNS jsonb
LANGUAGE plpgsql VOLATILE PARALLEL SAFE AS
$$
DECLARE
result jsonb;
BEGIN
IF _level < 1 THEN
RETURN '[]'::jsonb;
END IF;
EXECUTE 'CREATE TEMP TABLE IF NOT EXISTS t_item ON COMMIT DROP AS'
|| ' SELECT "Item".*, "Item".created_at at time zone ''UTC'' AS "createdAt", "Item".updated_at at time zone ''UTC'' AS "updatedAt", '
|| ' "Item"."invoicePaidAt" at time zone ''UTC'' AS "invoicePaidAtUTC", to_jsonb(users.*) || jsonb_build_object(''meMute'', "Mute"."mutedId" IS NOT NULL) AS user, '
|| ' COALESCE("ItemAct"."meMsats", 0) AS "meMsats", COALESCE("ItemAct"."mePendingMsats", 0) as "mePendingMsats", COALESCE("ItemAct"."meDontLikeMsats", 0) AS "meDontLikeMsats", '
|| ' COALESCE("ItemAct"."meMcredits", 0) AS "meMcredits", COALESCE("ItemAct"."mePendingMcredits", 0) as "mePendingMcredits", '
|| ' "Bookmark"."itemId" IS NOT NULL AS "meBookmark", "ThreadSubscription"."itemId" IS NOT NULL AS "meSubscription", '
|| ' g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore" '
|| ' FROM "Item" '
|| ' JOIN users ON users.id = "Item"."userId" '
|| ' LEFT JOIN "Mute" ON "Mute"."muterId" = $5 AND "Mute"."mutedId" = "Item"."userId"'
|| ' LEFT JOIN "Bookmark" ON "Bookmark"."userId" = $5 AND "Bookmark"."itemId" = "Item".id '
|| ' LEFT JOIN "ThreadSubscription" ON "ThreadSubscription"."userId" = $5 AND "ThreadSubscription"."itemId" = "Item".id '
|| ' LEFT JOIN LATERAL ( '
|| ' SELECT "itemId", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND "InvoiceForward".id IS NOT NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "meMsats", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND "InvoiceForward".id IS NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "meMcredits", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS NOT DISTINCT FROM ''PENDING'' AND "InvoiceForward".id IS NOT NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "mePendingMsats", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS NOT DISTINCT FROM ''PENDING'' AND "InvoiceForward".id IS NULL AND (act = ''FEE'' OR act = ''TIP'')) AS "mePendingMcredits", '
|| ' sum("ItemAct".msats) FILTER (WHERE "invoiceActionState" IS DISTINCT FROM ''FAILED'' AND act = ''DONT_LIKE_THIS'') AS "meDontLikeMsats" '
|| ' FROM "ItemAct" '
|| ' LEFT JOIN "Invoice" ON "Invoice".id = "ItemAct"."invoiceId" '
|| ' LEFT JOIN "InvoiceForward" ON "InvoiceForward"."invoiceId" = "Invoice"."id" '
|| ' WHERE "ItemAct"."userId" = $5 '
|| ' AND "ItemAct"."itemId" = "Item".id '
|| ' GROUP BY "ItemAct"."itemId" '
|| ' ) "ItemAct" ON true '
|| ' LEFT JOIN hot_score_view g ON g.id = "Item".id '
|| ' WHERE "Item".path <@ (SELECT path FROM "Item" WHERE id = $1) ' || _where || ' '
USING _item_id, _level, _where, _order_by, _me_id, _global_seed;
EXECUTE ''
|| 'SELECT COALESCE(jsonb_agg(sub), ''[]''::jsonb) AS comments '
|| 'FROM ( '
|| ' SELECT "Item".*, item_comments_zaprank_with_me("Item".id, $6, $5, $2 - 1, $3, $4) AS comments '
|| ' FROM t_item "Item" '
|| ' WHERE "Item"."parentId" = $1 '
|| _order_by
|| ' ) sub'
INTO result USING _item_id, _level, _where, _order_by, _me_id, _global_seed;
RETURN result;
END
$$;
CREATE OR REPLACE FUNCTION item_comments(_item_id int, _level int, _where text, _order_by text)
RETURNS jsonb
LANGUAGE plpgsql VOLATILE PARALLEL SAFE AS
$$
DECLARE
result jsonb;
BEGIN
IF _level < 1 THEN
RETURN '[]'::jsonb;
END IF;
EXECUTE 'CREATE TEMP TABLE IF NOT EXISTS t_item ON COMMIT DROP AS'
|| ' SELECT "Item".*, "Item".created_at at time zone ''UTC'' AS "createdAt", "Item".updated_at at time zone ''UTC'' AS "updatedAt", '
|| ' "Item"."invoicePaidAt" at time zone ''UTC'' AS "invoicePaidAtUTC", '
|| ' to_jsonb(users.*) as user, '
|| ' g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore" '
|| ' FROM "Item" '
|| ' JOIN users ON users.id = "Item"."userId" '
|| ' LEFT JOIN hot_score_view g ON g.id = "Item".id '
|| ' WHERE "Item".path <@ (SELECT path FROM "Item" WHERE id = $1) ' || _where
USING _item_id, _level, _where, _order_by;
EXECUTE ''
|| 'SELECT COALESCE(jsonb_agg(sub), ''[]''::jsonb) AS comments '
|| 'FROM ( '
|| ' SELECT "Item".*, item_comments("Item".id, $2 - 1, $3, $4) AS comments '
|| ' FROM t_item "Item"'
|| ' WHERE "Item"."parentId" = $1 '
|| _order_by
|| ' ) sub'
INTO result USING _item_id, _level, _where, _order_by;
RETURN result;
END
$$;
-- add limit and offset
CREATE OR REPLACE FUNCTION item_comments_limited(
_item_id int, _limit int, _offset int, _grandchild_limit int,
_level int, _where text, _order_by text)
RETURNS jsonb
LANGUAGE plpgsql VOLATILE PARALLEL SAFE AS
$$
DECLARE
result jsonb;
BEGIN
IF _level < 1 THEN
RETURN '[]'::jsonb;
END IF;
EXECUTE 'CREATE TEMP TABLE IF NOT EXISTS t_item ON COMMIT DROP AS '
|| 'WITH RECURSIVE base AS ( '
|| ' (SELECT "Item".*, 1 as level, ROW_NUMBER() OVER () as rn '
|| ' FROM "Item" '
|| ' WHERE "Item"."parentId" = $1 '
|| _order_by || ' '
|| ' LIMIT $2 '
|| ' OFFSET $3) '
|| ' UNION ALL '
|| ' (SELECT "Item".*, b.level + 1, ROW_NUMBER() OVER (PARTITION BY "Item"."parentId" ' || _order_by || ') '
|| ' FROM "Item" '
|| ' JOIN base b ON "Item"."parentId" = b.id '
|| ' LEFT JOIN hot_score_view g ON g.id = "Item".id '
|| ' WHERE b.level < $5 AND (b.level = 1 OR b.rn <= $4)) '
|| ') '
|| 'SELECT "Item".*, "Item".created_at at time zone ''UTC'' AS "createdAt", "Item".updated_at at time zone ''UTC'' AS "updatedAt", '
|| ' "Item"."invoicePaidAt" at time zone ''UTC'' AS "invoicePaidAtUTC", '
|| ' to_jsonb(users.*) as user, '
|| ' g.hot_score AS "hotScore", g.sub_hot_score AS "subHotScore" '
|| 'FROM base "Item" '
|| 'JOIN users ON users.id = "Item"."userId" '
|| 'LEFT JOIN hot_score_view g ON g.id = "Item".id '
|| 'WHERE ("Item".level = 1 OR "Item".rn <= $4 - "Item".level + 2) ' || _where
USING _item_id, _limit, _offset, _grandchild_limit, _level, _where, _order_by;
EXECUTE ''
|| 'SELECT COALESCE(jsonb_agg(sub), ''[]''::jsonb) AS comments '
|| 'FROM ( '
|| ' SELECT "Item".*, item_comments_limited("Item".id, $2, $3, $4, $5 - 1, $6, $7) AS comments '
|| ' FROM t_item "Item" '
|| ' WHERE "Item"."parentId" = $1 '
|| _order_by
|| ' ) sub'
INTO result USING _item_id, _limit, _offset, _grandchild_limit, _level, _where, _order_by;
RETURN result;
END
$$;

177
prisma/schema.prisma
View File

@ -148,6 +148,7 @@ model User {
directReceive Boolean @default(true)
DirectPaymentReceived DirectPayment[] @relation("DirectPaymentReceived")
DirectPaymentSent DirectPayment[] @relation("DirectPaymentSent")
UserSubTrust UserSubTrust[]
@@index([photoId])
@@index([createdAt], map: "users.created_at_index")
@ -184,6 +185,21 @@ model OneDayReferral {
@@index([type, typeId])
}
model UserSubTrust {
subName String @db.Citext
userId Int
zapPostTrust Float @default(0)
subZapPostTrust Float @default(0)
zapCommentTrust Float @default(0)
subZapCommentTrust Float @default(0)
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
sub Sub @relation(fields: [subName], references: [name], onDelete: Cascade)
@@id([userId, subName])
}
enum WalletType {
LIGHTNING_ADDRESS
LND
@ -498,85 +514,87 @@ model Message {
/// This model contains an expression index which requires additional setup for migrations. Visit https://pris.ly/d/expression-indexes for more info.
model Item {
id Int @id @default(autoincrement())
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
title String?
text String?
url String?
userId Int
parentId Int?
path Unsupported("ltree")?
pinId Int?
latitude Float?
location String?
longitude Float?
maxBid Int?
maxSalary Int?
minSalary Int?
remote Boolean?
subName String? @db.Citext
statusUpdatedAt DateTime?
status Status @default(ACTIVE)
company String?
weightedVotes Float @default(0)
boost Int @default(0)
oldBoost Int @default(0)
pollCost Int?
paidImgLink Boolean @default(false)
commentMsats BigInt @default(0)
commentMcredits BigInt @default(0)
lastCommentAt DateTime?
lastZapAt DateTime?
ncomments Int @default(0)
nDirectComments Int @default(0)
msats BigInt @default(0)
mcredits BigInt @default(0)
cost Int @default(0)
weightedDownVotes Float @default(0)
bio Boolean @default(false)
freebie Boolean @default(false)
deletedAt DateTime?
otsFile Bytes?
otsHash String?
imgproxyUrls Json?
bounty Int?
noteId String? @unique(map: "Item.noteId_unique")
rootId Int?
bountyPaidTo Int[]
upvotes Int @default(0)
weightedComments Float @default(0)
Bookmark Bookmark[]
parent Item? @relation("ParentChildren", fields: [parentId], references: [id])
children Item[] @relation("ParentChildren")
pin Pin? @relation(fields: [pinId], references: [id])
root Item? @relation("RootDescendant", fields: [rootId], references: [id])
descendants Item[] @relation("RootDescendant")
sub Sub? @relation(fields: [subName], references: [name], onDelete: Cascade, onUpdate: Cascade)
user User @relation("UserItems", fields: [userId], references: [id], onDelete: Cascade)
itemActs ItemAct[]
mentions Mention[]
itemReferrers ItemMention[] @relation("referrer")
itemReferees ItemMention[] @relation("referee")
pollOptions PollOption[]
PollVote PollVote[]
threadSubscriptions ThreadSubscription[]
User User[]
itemForwards ItemForward[]
itemUploads ItemUpload[]
uploadId Int?
invoiceId Int?
invoiceActionState InvoiceActionState?
invoicePaidAt DateTime?
outlawed Boolean @default(false)
apiKey Boolean @default(false)
pollExpiresAt DateTime?
Ancestors Reply[] @relation("AncestorReplyItem")
Replies Reply[]
Reminder Reminder[]
invoice Invoice? @relation(fields: [invoiceId], references: [id], onDelete: SetNull)
PollBlindVote PollBlindVote[]
ItemUserAgg ItemUserAgg[]
id Int @id @default(autoincrement())
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
title String?
text String?
url String?
userId Int
parentId Int?
path Unsupported("ltree")?
pinId Int?
latitude Float?
location String?
longitude Float?
maxBid Int?
maxSalary Int?
minSalary Int?
remote Boolean?
subName String? @db.Citext
statusUpdatedAt DateTime?
status Status @default(ACTIVE)
company String?
weightedVotes Float @default(0)
subWeightedVotes Float @default(0)
boost Int @default(0)
oldBoost Int @default(0)
pollCost Int?
paidImgLink Boolean @default(false)
commentMsats BigInt @default(0)
commentMcredits BigInt @default(0)
lastCommentAt DateTime?
lastZapAt DateTime?
ncomments Int @default(0)
nDirectComments Int @default(0)
msats BigInt @default(0)
mcredits BigInt @default(0)
cost Int @default(0)
weightedDownVotes Float @default(0)
subWeightedDownVotes Float @default(0)
bio Boolean @default(false)
freebie Boolean @default(false)
deletedAt DateTime?
otsFile Bytes?
otsHash String?
imgproxyUrls Json?
bounty Int?
noteId String? @unique(map: "Item.noteId_unique")
rootId Int?
bountyPaidTo Int[]
upvotes Int @default(0)
weightedComments Float @default(0)
Bookmark Bookmark[]
parent Item? @relation("ParentChildren", fields: [parentId], references: [id])
children Item[] @relation("ParentChildren")
pin Pin? @relation(fields: [pinId], references: [id])
root Item? @relation("RootDescendant", fields: [rootId], references: [id])
descendants Item[] @relation("RootDescendant")
sub Sub? @relation(fields: [subName], references: [name], onDelete: Cascade, onUpdate: Cascade)
user User @relation("UserItems", fields: [userId], references: [id], onDelete: Cascade)
itemActs ItemAct[]
mentions Mention[]
itemReferrers ItemMention[] @relation("referrer")
itemReferees ItemMention[] @relation("referee")
pollOptions PollOption[]
PollVote PollVote[]
threadSubscriptions ThreadSubscription[]
User User[]
itemForwards ItemForward[]
itemUploads ItemUpload[]
uploadId Int?
invoiceId Int?
invoiceActionState InvoiceActionState?
invoicePaidAt DateTime?
outlawed Boolean @default(false)
apiKey Boolean @default(false)
pollExpiresAt DateTime?
Ancestors Reply[] @relation("AncestorReplyItem")
Replies Reply[]
Reminder Reminder[]
invoice Invoice? @relation(fields: [invoiceId], references: [id], onDelete: SetNull)
PollBlindVote PollBlindVote[]
ItemUserAgg ItemUserAgg[]
@@index([uploadId])
@@index([lastZapAt])
@ -760,6 +778,7 @@ model Sub {
MuteSub MuteSub[]
SubSubscription SubSubscription[]
TerritoryTransfer TerritoryTransfer[]
UserSubTrust UserSubTrust[]
@@index([parentName])
@@index([createdAt])

272
worker/trust.js
View File

@ -1,38 +1,89 @@
import * as math from 'mathjs'
import { USER_ID, SN_ADMIN_IDS } from '@/lib/constants'
import { USER_ID } from '@/lib/constants'
import { Prisma } from '@prisma/client'
import { initialTrust, GLOBAL_SEEDS } from '@/api/paidAction/lib/territory'
export async function trust ({ boss, models }) {
try {
console.time('trust')
console.timeLog('trust', 'getting graph')
const graph = await getGraph(models)
console.timeLog('trust', 'computing trust')
const [vGlobal, mPersonal] = await trustGivenGraph(graph)
console.timeLog('trust', 'storing trust')
await storeTrust(models, graph, vGlobal, mPersonal)
} finally {
console.timeEnd('trust')
}
}
const MAX_DEPTH = 10
const MAX_DEPTH = 40
const MAX_TRUST = 1
const MIN_SUCCESS = 1
const MIN_SUCCESS = 0
// https://en.wikipedia.org/wiki/Normal_distribution#Quantile_function
const Z_CONFIDENCE = 6.109410204869 // 99.9999999% confidence
const GLOBAL_ROOT = 616
const SEED_WEIGHT = 1.0
const SEED_WEIGHT = 0.83
const AGAINST_MSAT_MIN = 1000
const MSAT_MIN = 20001 // 20001 is the minimum for a tip to be counted in trust
const SIG_DIFF = 0.1 // need to differ by at least 10 percent
const MSAT_MIN = 1001 // 20001 is the minimum for a tip to be counted in trust
const INDEPENDENCE_THRESHOLD = 50 // how many zappers are needed to consider a sub independent
const IRRELEVANT_CUMULATIVE_TRUST = 0.001 // if a user has less than this amount of cumulative trust, they are irrelevant
// for each subName, we'll need to get two graphs
// one for comments and one for posts
// then we'll need to do two trust calculations on each graph
// one with global seeds and one with subName seeds
export async function trust ({ boss, models }) {
console.time('trust')
const territories = await models.sub.findMany({
where: {
status: 'ACTIVE'
}
})
for (const territory of territories) {
const seeds = GLOBAL_SEEDS.includes(territory.userId) ? GLOBAL_SEEDS : GLOBAL_SEEDS.concat(territory.userId)
try {
console.timeLog('trust', `getting post graph for ${territory.name}`)
const postGraph = await getGraph(models, territory.name, true, seeds)
console.timeLog('trust', `getting comment graph for ${territory.name}`)
const commentGraph = await getGraph(models, territory.name, false, seeds)
console.timeLog('trust', `computing global post trust for ${territory.name}`)
const vGlobalPost = await trustGivenGraph(postGraph)
console.timeLog('trust', `computing global comment trust for ${territory.name}`)
const vGlobalComment = await trustGivenGraph(commentGraph)
console.timeLog('trust', `computing sub post trust for ${territory.name}`)
const vSubPost = await trustGivenGraph(postGraph, postGraph.length > INDEPENDENCE_THRESHOLD ? [territory.userId] : seeds)
console.timeLog('trust', `computing sub comment trust for ${territory.name}`)
const vSubComment = await trustGivenGraph(commentGraph, commentGraph.length > INDEPENDENCE_THRESHOLD ? [territory.userId] : seeds)
console.timeLog('trust', `storing trust for ${territory.name}`)
let results = reduceVectors(territory.name, {
zapPostTrust: {
graph: postGraph,
vector: vGlobalPost
},
subZapPostTrust: {
graph: postGraph,
vector: vSubPost
},
zapCommentTrust: {
graph: commentGraph,
vector: vGlobalComment
},
subZapCommentTrust: {
graph: commentGraph,
vector: vSubComment
}
})
if (results.length === 0) {
console.timeLog('trust', `no results for ${territory.name} - adding seeds`)
results = initialTrust({ name: territory.name, userId: territory.userId })
}
await storeTrust(models, territory.name, results)
} catch (e) {
console.error(`error computing trust for ${territory.name}:`, e)
} finally {
console.timeLog('trust', `finished computing trust for ${territory.name}`)
}
}
console.timeEnd('trust')
}
/*
Given a graph and start this function returns an object where
the keys are the node id and their value is the trust of that node
*/
function trustGivenGraph (graph) {
// I'm going to need to send subName, and multiply by a vector instead of a matrix
function trustGivenGraph (graph, seeds = GLOBAL_SEEDS) {
console.timeLog('trust', `creating matrix of size ${graph.length} x ${graph.length}`)
// empty matrix of proper size nstackers x nstackers
let mat = math.zeros(graph.length, graph.length, 'sparse')
const mat = math.zeros(graph.length, graph.length, 'sparse')
// create a map of user id to position in matrix
const posByUserId = {}
@ -54,54 +105,57 @@ function trustGivenGraph (graph) {
// perform random walk over trust matrix
// the resulting matrix columns represent the trust a user (col) has for each other user (rows)
// XXX this scales N^3 and mathjs is slow
let matT = math.transpose(mat)
const original = matT.clone()
for (let i = 0; i < MAX_DEPTH; i++) {
console.timeLog('trust', `matrix multiply ${i}`)
matT = math.multiply(original, matT)
matT = math.add(math.multiply(1 - SEED_WEIGHT, matT), math.multiply(SEED_WEIGHT, original))
const matT = math.transpose(mat)
const vTrust = math.zeros(graph.length)
for (const seed of seeds) {
vTrust.set([posByUserId[seed], 0], 1.0 / seeds.length)
}
let result = vTrust.clone()
console.timeLog('trust', 'matrix multiply')
for (let i = 0; i < MAX_DEPTH; i++) {
result = math.multiply(matT, result)
result = math.add(math.multiply(1 - SEED_WEIGHT, result), math.multiply(SEED_WEIGHT, vTrust))
}
result = math.squeeze(result)
console.timeLog('trust', 'transforming result')
const seedIdxs = SN_ADMIN_IDS.map(id => posByUserId[id])
const isOutlier = (fromIdx, idx) => [...seedIdxs, fromIdx].includes(idx)
const sqapply = (mat, fn) => {
let idx = 0
return math.squeeze(math.apply(mat, 1, d => {
const filtered = math.filter(d, (val, fidx) => {
return val !== 0 && !isOutlier(idx, fidx[0])
})
idx++
if (filtered.length === 0) return 0
return fn(filtered)
}))
const seedIdxs = seeds.map(id => posByUserId[id])
const filterZeroAndSeed = (val, idx) => {
return val !== 0 && !seedIdxs.includes(idx[0])
}
const filterSeed = (val, idx) => {
return !seedIdxs.includes(idx[0])
}
const sqapply = (vec, filterFn, fn) => {
// if the vector is smaller than the seeds, don't filter
const filtered = vec.size()[0] > seeds.length ? math.filter(vec, filterFn) : vec
if (filtered.size()[0] === 0) return 0
return fn(filtered)
}
console.timeLog('trust', 'normalizing')
console.timeLog('trust', 'stats')
mat = math.transpose(matT)
const std = sqapply(mat, math.std) // math.squeeze(math.std(mat, 1))
const mean = sqapply(mat, math.mean) // math.squeeze(math.mean(mat, 1))
const zscore = math.map(mat, (val, idx) => {
const zstd = math.subset(std, math.index(idx[0], 0))
const zmean = math.subset(mean, math.index(idx[0], 0))
return zstd ? (val - zmean) / zstd : 0
const std = sqapply(result, filterZeroAndSeed, math.std) // math.squeeze(math.std(mat, 1))
const mean = sqapply(result, filterZeroAndSeed, math.mean) // math.squeeze(math.mean(mat, 1))
console.timeLog('trust', 'std', std)
console.timeLog('trust', 'mean', mean)
const zscore = math.map(result, (val) => {
if (std === 0) return 0
return (val - mean) / std
})
console.timeLog('trust', 'minmax')
const min = sqapply(zscore, math.min) // math.squeeze(math.min(zscore, 1))
const max = sqapply(zscore, math.max) // math.squeeze(math.max(zscore, 1))
const mPersonal = math.map(zscore, (val, idx) => {
const zmin = math.subset(min, math.index(idx[0], 0))
const zmax = math.subset(max, math.index(idx[0], 0))
const zrange = zmax - zmin
if (val > zmax) return MAX_TRUST
return zrange ? (val - zmin) / zrange : 0
const min = sqapply(zscore, filterSeed, math.min) // math.squeeze(math.min(zscore, 1))
const max = sqapply(zscore, filterSeed, math.max) // math.squeeze(math.max(zscore, 1))
console.timeLog('trust', 'min', min)
console.timeLog('trust', 'max', max)
const normalized = math.map(zscore, (val) => {
const zrange = max - min
if (val > max) return MAX_TRUST
return zrange ? (val - min) / zrange : 0
})
const vGlobal = math.squeeze(math.row(mPersonal, posByUserId[GLOBAL_ROOT]))
return [vGlobal, mPersonal]
return normalized
}
/*
@ -111,7 +165,9 @@ function trustGivenGraph (graph) {
...
]
*/
async function getGraph (models) {
// I'm going to want to send subName to this function
// and whether it's for comments or posts
async function getGraph (models, subName, postTrust = true, seeds = GLOBAL_SEEDS) {
return await models.$queryRaw`
SELECT id, json_agg(json_build_object(
'node', oid,
@ -124,10 +180,16 @@ async function getGraph (models) {
sum("ItemAct".msats) as user_msats
FROM "ItemAct"
JOIN "Item" ON "Item".id = "ItemAct"."itemId" AND "ItemAct".act IN ('FEE', 'TIP', 'DONT_LIKE_THIS')
AND "Item"."parentId" IS NULL AND NOT "Item".bio AND "Item"."userId" <> "ItemAct"."userId"
AND NOT "Item".bio AND "Item"."userId" <> "ItemAct"."userId"
AND ${postTrust
? Prisma.sql`"Item"."parentId" IS NULL AND "Item"."subName" = ${subName}::TEXT`
: Prisma.sql`
"Item"."parentId" IS NOT NULL
JOIN "Item" root ON "Item"."rootId" = root.id AND root."subName" = ${subName}::TEXT`
}
JOIN users ON "ItemAct"."userId" = users.id AND users.id <> ${USER_ID.anon}
WHERE "ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID'
GROUP BY user_id, name, item_id, user_at, against
WHERE ("ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID')
GROUP BY user_id, users.name, item_id, user_at, against
HAVING CASE WHEN
"ItemAct".act = 'DONT_LIKE_THIS' THEN sum("ItemAct".msats) > ${AGAINST_MSAT_MIN}
ELSE sum("ItemAct".msats) > ${MSAT_MIN} END
@ -136,7 +198,7 @@ async function getGraph (models) {
SELECT a.user_id AS a_id, b.user_id AS b_id,
sum(CASE WHEN b.user_msats > a.user_msats THEN a.user_msats / b.user_msats::FLOAT ELSE b.user_msats / a.user_msats::FLOAT END) FILTER(WHERE a.act_at > b.act_at AND a.against = b.against) AS before,
sum(CASE WHEN b.user_msats > a.user_msats THEN a.user_msats / b.user_msats::FLOAT ELSE b.user_msats / a.user_msats::FLOAT END) FILTER(WHERE b.act_at > a.act_at AND a.against = b.against) AS after,
sum(log(1 + a.user_msats / 10000::float) + log(1 + b.user_msats / 10000::float)) FILTER(WHERE a.against <> b.against) AS disagree,
count(*) FILTER(WHERE a.against <> b.against) AS disagree,
b.user_vote_count AS b_total, a.user_vote_count AS a_total
FROM user_votes a
JOIN user_votes b ON a.item_id = b.item_id
@ -149,14 +211,9 @@ async function getGraph (models) {
confidence(before - disagree, b_total - after, ${Z_CONFIDENCE})
ELSE 0 END AS trust
FROM user_pair
WHERE NOT (b_id = ANY (${SN_ADMIN_IDS}))
UNION ALL
SELECT a_id AS id, seed_id AS oid, ${MAX_TRUST}::numeric as trust
FROM user_pair, unnest(${SN_ADMIN_IDS}::int[]) seed_id
GROUP BY a_id, a_total, seed_id
UNION ALL
SELECT a_id AS id, a_id AS oid, ${MAX_TRUST}::float as trust
FROM user_pair
SELECT seed_id AS id, seed_id AS oid, 0 AS trust
FROM unnest(${seeds}::int[]) seed_id
)
SELECT id, oid, trust, sum(trust) OVER (PARTITION BY id) AS total_trust
FROM trust_pairs
@ -165,46 +222,45 @@ async function getGraph (models) {
ORDER BY id ASC`
}
async function storeTrust (models, graph, vGlobal, mPersonal) {
// convert nodeTrust into table literal string
let globalValues = ''
let personalValues = ''
vGlobal.forEach((val, [idx]) => {
if (isNaN(val)) return
if (globalValues) globalValues += ','
globalValues += `(${graph[idx].id}, ${val}::FLOAT)`
if (personalValues) personalValues += ','
personalValues += `(${GLOBAL_ROOT}, ${graph[idx].id}, ${val}::FLOAT)`
})
function reduceVectors (subName, fieldGraphVectors) {
function reduceVector (field, graph, vector, result = {}) {
vector.forEach((val, [idx]) => {
if (isNaN(val) || val <= 0) return
result[graph[idx].id] = {
...result[graph[idx].id],
subName,
userId: graph[idx].id,
[field]: val
}
})
return result
}
math.forEach(mPersonal, (val, [fromIdx, toIdx]) => {
const globalVal = vGlobal.get([toIdx, 0])
if (isNaN(val) || val - globalVal <= SIG_DIFF) return
if (personalValues) personalValues += ','
personalValues += `(${graph[fromIdx].id}, ${graph[toIdx].id}, ${val}::FLOAT)`
})
let result = {}
for (const field in fieldGraphVectors) {
result = reduceVector(field, fieldGraphVectors[field].graph, fieldGraphVectors[field].vector, result)
}
// return only the users with trust > 0
return Object.values(result).filter(s =>
Object.keys(fieldGraphVectors).reduce(
(acc, key) => acc + (s[key] ?? 0),
0
) > IRRELEVANT_CUMULATIVE_TRUST
)
}
async function storeTrust (models, subName, results) {
console.timeLog('trust', `storing trust for ${subName} with ${results.length} users`)
// update the trust of each user in graph
await models.$transaction([
models.$executeRaw`UPDATE users SET trust = 0`,
models.$executeRawUnsafe(
`UPDATE users
SET trust = g.trust
FROM (values ${globalValues}) g(id, trust)
WHERE users.id = g.id`),
models.$executeRawUnsafe(
`INSERT INTO "Arc" ("fromId", "toId", "zapTrust")
SELECT id, oid, trust
FROM (values ${personalValues}) g(id, oid, trust)
ON CONFLICT ("fromId", "toId") DO UPDATE SET "zapTrust" = EXCLUDED."zapTrust"`
),
// select all arcs that don't exist in personalValues and delete them
models.$executeRawUnsafe(
`DELETE FROM "Arc"
WHERE ("fromId", "toId") NOT IN (
SELECT id, oid
FROM (values ${personalValues}) g(id, oid, trust)
)`
)
models.userSubTrust.deleteMany({
where: {
subName
}
}),
models.userSubTrust.createMany({
data: results
})
])
}

2
worker/views.js
View File

@ -32,7 +32,7 @@ export async function rankViews () {
const models = createPrisma({ connectionParams: { connection_limit: 1 } })
try {
for (const view of ['zap_rank_personal_view']) {
for (const view of ['hot_score_view']) {
await models.$queryRawUnsafe(`REFRESH MATERIALIZED VIEW CONCURRENTLY ${view}`)
}
} finally {