From c6581b2cb1814aed8a136949c9f4ad71727f23f2 Mon Sep 17 00:00:00 2001 From: keyan Date: Tue, 16 Jul 2024 13:41:44 -0500 Subject: [PATCH 1/3] allow db connection limit and timeout configuration --- .env.development | 3 +++ .env.production | 5 ++++- api/models/index.js | 14 ++++---------- lib/create-prisma.js | 28 ++++++++++++++++++++++++++++ worker/earn.js | 4 ++-- worker/index.js | 6 ++++-- worker/views.js | 6 +++--- 7 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 lib/create-prisma.js diff --git a/.env.development b/.env.development index 5cde68a8..4aa4763f 100644 --- a/.env.development +++ b/.env.development @@ -92,6 +92,9 @@ OPENSEARCH_MODEL_ID= # prisma db url DATABASE_URL="postgresql://sn:password@db:5432/stackernews?schema=public" +DB_APP_CONNECTION_LIMIT=2 +DB_WORKER_CONNECTION_LIMIT=2 +DB_TRANSACTION_TIMEOUT=5000 # polling intervals NEXT_PUBLIC_FAST_POLL_INTERVAL=1000 diff --git a/.env.production b/.env.production index 72ab1011..ca5c5bb5 100644 --- a/.env.production +++ b/.env.production @@ -18,4 +18,7 @@ NEXT_PUBLIC_EXTRA_LONG_POLL_INTERVAL=300000 NEXT_PUBLIC_URL=https://stacker.news TOR_PROXY=http://127.0.0.1:7050/ PRISMA_SLOW_LOGS_MS=50 -GRAPHQL_SLOW_LOGS_MS=50 \ No newline at end of file +GRAPHQL_SLOW_LOGS_MS=50 +DB_APP_CONNECTION_LIMIT=4 +DB_WORKER_CONNECTION_LIMIT=2 +DB_TRANSACTION_TIMEOUT=10000 \ No newline at end of file diff --git a/api/models/index.js b/api/models/index.js index df098bda..d050e0a2 100644 --- a/api/models/index.js +++ b/api/models/index.js @@ -1,18 +1,12 @@ -import { PrismaClient } from '@prisma/client' +import createPrisma from '@/lib/create-prisma' const prisma = global.prisma || (() => { console.log('initing prisma') - const prisma = new PrismaClient({ - log: [{ level: 'query', emit: 'event' }, 'warn', 'error'] - }) - prisma.$on('query', (e) => { - if (process.env.PRISMA_SLOW_LOGS_MS && e.duration > process.env.PRISMA_SLOW_LOGS_MS) { - console.log('Query: ' + e.query) - console.log('Params: ' + e.params) - console.log('Duration: ' + e.duration + 'ms') + return createPrisma({ + connectionParams: { + connection_limit: process.env.DB_APP_CONNECTION_LIMIT } }) - return prisma })() if (process.env.NODE_ENV === 'development') global.prisma = prisma diff --git a/lib/create-prisma.js b/lib/create-prisma.js new file mode 100644 index 00000000..d5d3af27 --- /dev/null +++ b/lib/create-prisma.js @@ -0,0 +1,28 @@ +import { PrismaClient } from '@prisma/client' + +export default function newPrismaClient ({ options = {}, connectionParams = {} }) { + const url = new URL(process.env.DATABASE_URL) + for (const [key, value] of Object.entries(connectionParams)) { + if (value === undefined) continue + url.searchParams.set(key, value) + } + + const prisma = new PrismaClient({ + datasourceUrl: url.toString(), + ...options, + transactionOptions: { + timeout: process.env.DB_TRANSACTION_TIMEOUT ? parseInt(process.env.DB_TRANSACTION_TIMEOUT) : undefined, + ...options.transactionOptions + } + }) + + prisma.$on('query', (e) => { + if (process.env.PRISMA_SLOW_LOGS_MS && e.duration > process.env.PRISMA_SLOW_LOGS_MS) { + console.log('Query: ' + e.query) + console.log('Params: ' + e.params) + console.log('Duration: ' + e.duration + 'ms') + } + }) + + return prisma +} diff --git a/worker/earn.js b/worker/earn.js index ed98e489..52ef63dd 100644 --- a/worker/earn.js +++ b/worker/earn.js @@ -1,5 +1,5 @@ import { notifyEarner } from '@/lib/webPush.js' -import { PrismaClient } from '@prisma/client' +import createPrisma from '@/lib/create-prisma.js' import { proportions } from '@/lib/madness.js' import { SN_NO_REWARDS_IDS } from '@/lib/constants.js' @@ -7,7 +7,7 @@ const TOTAL_UPPER_BOUND_MSATS = 1_000_000_000 export async function earn ({ name }) { // grab a greedy connection - const models = new PrismaClient() + const models = createPrisma({ connectionParams: { connection_limit: 1 } }) try { // compute how much sn earned yesterday diff --git a/worker/index.js b/worker/index.js index c601ad04..738ac5a6 100644 --- a/worker/index.js +++ b/worker/index.js @@ -1,6 +1,6 @@ import PgBoss from 'pg-boss' import nextEnv from '@next/env' -import { PrismaClient } from '@prisma/client' +import createPrisma from '@/lib/create-prisma.js' import { autoDropBolt11s, checkInvoice, checkPendingDeposits, checkPendingWithdrawals, finalizeHodlInvoice, subscribeToWallet @@ -34,7 +34,9 @@ loadEnvConfig('.', process.env.NODE_ENV === 'development') async function work () { const boss = new PgBoss(process.env.DATABASE_URL) - const models = new PrismaClient() + const models = createPrisma({ + connectionParams: { connection_limit: process.env.DB_WORKER_CONNECTION_LIMIT } + }) const apollo = new ApolloClient({ link: new HttpLink({ diff --git a/worker/views.js b/worker/views.js index 99ce219d..d9acf252 100644 --- a/worker/views.js +++ b/worker/views.js @@ -1,4 +1,4 @@ -import { PrismaClient } from '@prisma/client' +import createPrisma from '@/lib/create-prisma.js' const viewPrefixes = ['reg_growth', 'spender_growth', 'item_growth', 'spending_growth', 'stackers_growth', 'stacking_growth', 'user_stats', 'sub_stats'] @@ -6,7 +6,7 @@ const viewPrefixes = ['reg_growth', 'spender_growth', 'item_growth', 'spending_g // this is intended to be run everyday after midnight CT export async function views ({ data: { period } = { period: 'days' } }) { // grab a greedy connection - const models = new PrismaClient() + const models = createPrisma({ connectionParams: { connection_limit: 1 } }) try { // these views are bespoke so we can't use the loop @@ -29,7 +29,7 @@ export async function views ({ data: { period } = { period: 'days' } }) { // this should be run regularly ... like, every 5 minutes export async function rankViews () { // grab a greedy connection - const models = new PrismaClient() + const models = createPrisma({ connectionParams: { connection_limit: 1 } }) try { for (const view of ['zap_rank_personal_view']) { From 98efe763a07fee69821f5192373e7d1c7e1c2861 Mon Sep 17 00:00:00 2001 From: keyan Date: Tue, 16 Jul 2024 13:44:58 -0500 Subject: [PATCH 2/3] remove done referral todos --- api/paidAction/itemCreate.js | 2 -- api/paidAction/itemUpdate.js | 2 -- api/paidAction/zap.js | 1 - 3 files changed, 5 deletions(-) diff --git a/api/paidAction/itemCreate.js b/api/paidAction/itemCreate.js index 9c099f84..2ed2f634 100644 --- a/api/paidAction/itemCreate.js +++ b/api/paidAction/itemCreate.js @@ -194,8 +194,6 @@ export async function onPaid ({ invoice, id }, context) { INSERT INTO pgboss.job (name, data, retrylimit, retrybackoff, startafter) VALUES ('imgproxy', jsonb_build_object('id', ${item.id}::INTEGER), 21, true, now() + interval '5 seconds')` - // TODO: referals for boost - if (item.parentId) { // denormalize ncomments, lastCommentAt, and "weightedComments" for ancestors, and insert into reply table await tx.$executeRaw` diff --git a/api/paidAction/itemUpdate.js b/api/paidAction/itemUpdate.js index 15b2420a..a0dfaf21 100644 --- a/api/paidAction/itemUpdate.js +++ b/api/paidAction/itemUpdate.js @@ -131,8 +131,6 @@ export async function perform (args, context) { await performBotBehavior(args, context) - // TODO: referals for boost - // compare timestamps to only notify if mention or item referral was just created to avoid duplicates on edits for (const { userId, createdAt } of item.mentions) { if (item.updatedAt.getTime() !== createdAt.getTime()) continue diff --git a/api/paidAction/zap.js b/api/paidAction/zap.js index 40dce138..969e2039 100644 --- a/api/paidAction/zap.js +++ b/api/paidAction/zap.js @@ -143,7 +143,6 @@ export async function onPaid ({ invoice, actIds }, { models, tx }) { FROM zapped WHERE "Item".path @> zapped.path AND "Item".id <> zapped.id` - // TODO: referrals notifyZapped({ models, item }).catch(console.error) } From 3a748b8d38dd6e5321c6679b8bb221caa1bd9cea Mon Sep 17 00:00:00 2001 From: keyan Date: Tue, 16 Jul 2024 13:54:21 -0500 Subject: [PATCH 3/3] exclude saloon from rewards --- lib/constants.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/constants.js b/lib/constants.js index c19c3c49..525d6698 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -43,10 +43,11 @@ export const USER_ID = { sn: 4502, anon: 27, ad: 9, - delete: 106 + delete: 106, + saloon: 17226 } export const SN_USER_IDS = [USER_ID.k00b, USER_ID.ek, USER_ID.sn] -export const SN_NO_REWARDS_IDS = [USER_ID.anon, USER_ID.sn] +export const SN_NO_REWARDS_IDS = [USER_ID.anon, USER_ID.sn, USER_ID.saloon] export const ANON_INV_PENDING_LIMIT = 1000 export const ANON_BALANCE_LIMIT_MSATS = 0 // disable export const MAX_POLL_NUM_CHOICES = 10