ekzyis
/
stacker.news
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

search index functions

This commit is contained in:
keyan 2022-01-25 13:34:51 -06:00
parent d0403fc959
commit d413b49e24
7 changed files with 271 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
api/resolvers/item.js
View File

@ -182,6 +182,19 @@ export default {
pins
}
},
allItems: async (parent, { cursor }, { models }) => {
const decodedCursor = decodeCursor(cursor)
const items = await models.$queryRaw(`
${SELECT}
FROM "Item"
ORDER BY created_at DESC
OFFSET $1
LIMIT ${LIMIT}`, decodedCursor.offset)
return {
cursor: items.length === LIMIT ? nextCursorEncoded(decodedCursor) : null,
items
}
},
moreFlatComments: async (parent, { cursor, name, sort, within }, { me, models }) => {
const decodedCursor = decodeCursor(cursor)

5
api/search/index.js Normal file
View File

@ -0,0 +1,5 @@
import es from '@elastic/elasticsearch'
global.es ||= new es.Client()
export default global.es

2
api/typeDefs/item.js
View File

@ -8,6 +8,7 @@ export default gql`
comments(id: ID!, sort: String): [Item!]!
pageTitle(url: String!): String
dupes(url: String!): [Item!]
allItems(cursor: String): Items
}
type ItemActResult {
@ -39,6 +40,7 @@ export default gql`
type Item {
id: ID!
createdAt: String!
updatedAt: String!
title: String
url: String
text: String

133
package-lock.json generated
View File

@ -9,6 +9,8 @@
"version": "0.1.0",
"dependencies": {
"@apollo/client": "^3.4.15",
"@elastic/elasticsearch": "^7.16.0",
"@opensearch-project/opensearch": "^1.0.2",
"@prisma/client": "^2.25.0",
"apollo-server-micro": "^2.21.2",
"async-retry": "^1.3.1",
@ -18,6 +20,7 @@
"bolt11": "^1.3.4",
"bootstrap": "^4.6.0",
"clipboard-copy": "^4.0.1",
"cross-fetch": "^3.1.5",
"domino": "^2.1.6",
"formik": "^2.2.6",
"graphql": "^15.5.0",
@ -349,6 +352,25 @@
"node": ">=6.9.0"
}
},
"node_modules/@elastic/elasticsearch": {
"version": "7.16.0",
"resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.16.0.tgz",
"integrity": "sha512-lMY2MFZZFG3om7QNHninxZZOXYx3NdIUwEISZxqaI9dXPoL3DNhU31keqjvx1gN6T74lGXAzrRNP4ag8CJ/VXw==",
"dependencies": {
"debug": "^4.3.1",
"hpagent": "^0.1.1",
"ms": "^2.1.3",
"secure-json-parse": "^2.4.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@elastic/elasticsearch/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/@eslint/eslintrc": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
@ -773,6 +795,25 @@
"@napi-rs/triples": "^1.0.3"
}
},
"node_modules/@opensearch-project/opensearch": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-1.0.2.tgz",
"integrity": "sha512-gQ2CjbS7/pJ4A3IBWd+AD0KbCaAnE1Ur9baRxqM1NMH/7A8GQxwtVxx8raUOf4HZExkZZOUYBMzJgWM1OyELyw==",
"dependencies": {
"debug": "^4.3.1",
"hpagent": "^0.1.1",
"ms": "^2.1.3",
"secure-json-parse": "^2.4.0"
},
"engines": {
"node": ">=10"
}
},
"node_modules/@opensearch-project/opensearch/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/@panva/asn1.js": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@panva/asn1.js/-/asn1.js-1.0.0.tgz",
@ -3061,6 +3102,14 @@
"node": ">=0.8"
}
},
"node_modules/cross-fetch": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
"integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==",
"dependencies": {
"node-fetch": "2.6.7"
}
},
"node_modules/cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@ -4980,6 +5029,11 @@
"resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
"integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw=="
},
"node_modules/hpagent": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/hpagent/-/hpagent-0.1.2.tgz",
"integrity": "sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ=="
},
"node_modules/htmlparser2": {
"version": "3.8.3",
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz",
@ -7122,14 +7176,22 @@
"integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA=="
},
"node_modules/node-fetch": {
"version": "2.6.5",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
"integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
"version": "2.6.7",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/node-gyp-build": {
@ -9280,6 +9342,11 @@
"node": ">=10.0.0"
}
},
"node_modules/secure-json-parse": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.4.0.tgz",
"integrity": "sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg=="
},
"node_modules/semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@ -12166,6 +12233,24 @@
"to-fast-properties": "^2.0.0"
}
},
"@elastic/elasticsearch": {
"version": "7.16.0",
"resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.16.0.tgz",
"integrity": "sha512-lMY2MFZZFG3om7QNHninxZZOXYx3NdIUwEISZxqaI9dXPoL3DNhU31keqjvx1gN6T74lGXAzrRNP4ag8CJ/VXw==",
"requires": {
"debug": "^4.3.1",
"hpagent": "^0.1.1",
"ms": "^2.1.3",
"secure-json-parse": "^2.4.0"
},
"dependencies": {
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
}
}
},
"@eslint/eslintrc": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
@ -12462,6 +12547,24 @@
"@napi-rs/triples": "^1.0.3"
}
},
"@opensearch-project/opensearch": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-1.0.2.tgz",
"integrity": "sha512-gQ2CjbS7/pJ4A3IBWd+AD0KbCaAnE1Ur9baRxqM1NMH/7A8GQxwtVxx8raUOf4HZExkZZOUYBMzJgWM1OyELyw==",
"requires": {
"debug": "^4.3.1",
"hpagent": "^0.1.1",
"ms": "^2.1.3",
"secure-json-parse": "^2.4.0"
},
"dependencies": {
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
}
}
},
"@panva/asn1.js": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@panva/asn1.js/-/asn1.js-1.0.0.tgz",
@ -14339,6 +14442,14 @@
"luxon": "^1.28.0"
}
},
"cross-fetch": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
"integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==",
"requires": {
"node-fetch": "2.6.7"
}
},
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@ -15782,6 +15893,11 @@
"resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
"integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw=="
},
"hpagent": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/hpagent/-/hpagent-0.1.2.tgz",
"integrity": "sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ=="
},
"htmlparser2": {
"version": "3.8.3",
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz",
@ -17353,9 +17469,9 @@
"integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA=="
},
"node-fetch": {
"version": "2.6.5",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
"integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
"version": "2.6.7",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"requires": {
"whatwg-url": "^5.0.0"
}
@ -19021,6 +19137,11 @@
"node-gyp-build": "^4.2.0"
}
},
"secure-json-parse": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.4.0.tgz",
"integrity": "sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg=="
},
"semver": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",

3
package.json
View File

@ -10,6 +10,8 @@
},
"dependencies": {
"@apollo/client": "^3.4.15",
"@elastic/elasticsearch": "^7.16.0",
"@opensearch-project/opensearch": "^1.0.2",
"@prisma/client": "^2.25.0",
"apollo-server-micro": "^2.21.2",
"async-retry": "^1.3.1",
@ -19,6 +21,7 @@
"bolt11": "^1.3.4",
"bootstrap": "^4.6.0",
"clipboard-copy": "^4.0.1",
"cross-fetch": "^3.1.5",
"domino": "^2.1.6",
"formik": "^2.2.6",
"graphql": "^15.5.0",

25
worker/index.js
View File

@ -5,11 +5,32 @@ const { PrismaClient } = require('@prisma/client')
const { checkInvoice, checkWithdrawal } = require('./wallet')
const { repin } = require('./repin')
const { trust } = require('./trust')
const { ApolloClient, HttpLink, InMemoryCache } = require('@apollo/client')
const { indexItem, indexAllItems } = require('./search')
const fetch = require('cross-fetch')
async function work () {
const boss = new PgBoss(process.env.DATABASE_URL)
const models = new PrismaClient()
const args = { boss, models }
const apollo = new ApolloClient({
link: new HttpLink({
uri: `${process.env.SELF_URL}/api/graphql`,
fetch
}),
cache: new InMemoryCache(),
defaultOptions: {
watchQuery: {
fetchPolicy: 'network-only',
nextFetchPolicy: 'network-only'
},
query: {
fetchPolicy: 'network-only',
nextFetchPolicy: 'network-only'
}
}
})
const args = { boss, models, apollo }
boss.on('error', error => console.error(error))
@ -18,6 +39,8 @@ async function work () {
await boss.work('checkWithdrawal', checkWithdrawal(args))
await boss.work('repin-*', repin(args))
await boss.work('trust', trust(args))
await boss.work('indexItem', indexItem(args))
await boss.work('indexAllItems', indexAllItems(args))
console.log('working jobs')
}

97
worker/search.js Normal file
View File

@ -0,0 +1,97 @@
const { gql } = require('apollo-server-micro')
const es = require('@opensearch-project/opensearch')
const search = new es.Client({ node: 'http://localhost:9200' })
const ITEM_SEARCH_FIELDS = gql`
fragment ItemSearchFields on Item {
id
parentId
createdAt
updatedAt
title
text
url
user {
name
}
upvotes
sats
boost
ncomments
}`
async function _indexItem (item) {
console.log('indexing item', item.id)
try {
await search.index({
id: item.id,
index: 'item',
version: new Date(item.updatedAt).getTime(),
versionType: 'external_gte',
body: item
})
} catch (e) {
// ignore version conflict ...
if (e?.meta?.statusCode === 409) {
console.log('version conflict ignoring', item.id)
return
}
console.log(e)
throw e
}
console.log('done indexing item', item.id)
}
function indexItem ({ apollo }) {
return async function ({ data: { id } }) {
// 1. grab item from database
// could use apollo to avoid duping logic
// when grabbing sats and user name, etc
const { data: { item } } = await apollo.query({
query: gql`
${ITEM_SEARCH_FIELDS}
query Item {
item(id: ${id}) {
...ItemSearchFields
}
}`
})
// 2. index it with external version based on updatedAt
await _indexItem(item)
}
}
function indexAllItems ({ apollo }) {
return async function () {
// cursor over all items in the Item table
let items = []; let cursor = null
do {
// query for items
({ data: { allItems: { items, cursor } } } = await apollo.query({
query: gql`
${ITEM_SEARCH_FIELDS}
query AllItems($cursor: String) {
allItems(cursor: $cursor) {
items {
...ItemSearchFields
}
cursor
}
}`,
variables: { cursor }
}))
// for all items, index them
try {
items.forEach(_indexItem)
} catch (e) {
// ignore errors
console.log(e)
}
} while (cursor)
}
}
module.exports = { indexItem, indexAllItems }