search index functions

api/resolvers/item.js

@@ -182,6 +182,19 @@ export default {
         pins
       }
     },
+    allItems: async (parent, { cursor }, { models }) => {
+      const decodedCursor = decodeCursor(cursor)
+      const items = await models.$queryRaw(`
+        ${SELECT}
+        FROM "Item"
+        ORDER BY created_at DESC
+        OFFSET $1
+        LIMIT ${LIMIT}`, decodedCursor.offset)
+      return {
+        cursor: items.length === LIMIT ? nextCursorEncoded(decodedCursor) : null,
+        items
+      }
+    },
     moreFlatComments: async (parent, { cursor, name, sort, within }, { me, models }) => {
       const decodedCursor = decodeCursor(cursor)
 

api/search/index.js

@@ -0,0 +1,5 @@
+import es from '@elastic/elasticsearch'
+
+global.es ||= new es.Client()
+
+export default global.es

api/typeDefs/item.js

@@ -8,6 +8,7 @@ export default gql`
     comments(id: ID!, sort: String): [Item!]!
     pageTitle(url: String!): String
     dupes(url: String!): [Item!]
+    allItems(cursor: String): Items
   }
 
   type ItemActResult {
@@ -39,6 +40,7 @@ export default gql`
   type Item {
     id: ID!
     createdAt: String!
+    updatedAt: String!
     title: String
     url: String
     text: String

package-lock.json

@@ -9,6 +9,8 @@
       "version": "0.1.0",
       "dependencies": {
         "@apollo/client": "^3.4.15",
+        "@elastic/elasticsearch": "^7.16.0",
+        "@opensearch-project/opensearch": "^1.0.2",
         "@prisma/client": "^2.25.0",
         "apollo-server-micro": "^2.21.2",
         "async-retry": "^1.3.1",
@@ -18,6 +20,7 @@
         "bolt11": "^1.3.4",
         "bootstrap": "^4.6.0",
         "clipboard-copy": "^4.0.1",
+        "cross-fetch": "^3.1.5",
         "domino": "^2.1.6",
         "formik": "^2.2.6",
         "graphql": "^15.5.0",
@@ -349,6 +352,25 @@
         "node": ">=6.9.0"
       }
     },
+    "node_modules/@elastic/elasticsearch": {
+      "version": "7.16.0",
+      "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.16.0.tgz",
+      "integrity": "sha512-lMY2MFZZFG3om7QNHninxZZOXYx3NdIUwEISZxqaI9dXPoL3DNhU31keqjvx1gN6T74lGXAzrRNP4ag8CJ/VXw==",
+      "dependencies": {
+        "debug": "^4.3.1",
+        "hpagent": "^0.1.1",
+        "ms": "^2.1.3",
+        "secure-json-parse": "^2.4.0"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@elastic/elasticsearch/node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+    },
     "node_modules/@eslint/eslintrc": {
       "version": "0.4.3",
       "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
@@ -773,6 +795,25 @@
         "@napi-rs/triples": "^1.0.3"
       }
     },
+    "node_modules/@opensearch-project/opensearch": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-1.0.2.tgz",
+      "integrity": "sha512-gQ2CjbS7/pJ4A3IBWd+AD0KbCaAnE1Ur9baRxqM1NMH/7A8GQxwtVxx8raUOf4HZExkZZOUYBMzJgWM1OyELyw==",
+      "dependencies": {
+        "debug": "^4.3.1",
+        "hpagent": "^0.1.1",
+        "ms": "^2.1.3",
+        "secure-json-parse": "^2.4.0"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@opensearch-project/opensearch/node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+    },
     "node_modules/@panva/asn1.js": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/@panva/asn1.js/-/asn1.js-1.0.0.tgz",
@@ -3061,6 +3102,14 @@
         "node": ">=0.8"
       }
     },
+    "node_modules/cross-fetch": {
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
+      "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==",
+      "dependencies": {
+        "node-fetch": "2.6.7"
+      }
+    },
     "node_modules/cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -4980,6 +5029,11 @@
       "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
       "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw=="
     },
+    "node_modules/hpagent": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-0.1.2.tgz",
+      "integrity": "sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ=="
+    },
     "node_modules/htmlparser2": {
       "version": "3.8.3",
       "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz",
@@ -7122,14 +7176,22 @@
       "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA=="
     },
     "node_modules/node-fetch": {
-      "version": "2.6.5",
+      "version": "2.6.7",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
-      "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
+      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
       "dependencies": {
         "whatwg-url": "^5.0.0"
       },
       "engines": {
         "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
       }
     },
     "node_modules/node-gyp-build": {
@@ -9280,6 +9342,11 @@
         "node": ">=10.0.0"
       }
     },
+    "node_modules/secure-json-parse": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.4.0.tgz",
+      "integrity": "sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg=="
+    },
     "node_modules/semver": {
       "version": "6.3.0",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@@ -12166,6 +12233,24 @@
         "to-fast-properties": "^2.0.0"
       }
     },
+    "@elastic/elasticsearch": {
+      "version": "7.16.0",
+      "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-7.16.0.tgz",
+      "integrity": "sha512-lMY2MFZZFG3om7QNHninxZZOXYx3NdIUwEISZxqaI9dXPoL3DNhU31keqjvx1gN6T74lGXAzrRNP4ag8CJ/VXw==",
+      "requires": {
+        "debug": "^4.3.1",
+        "hpagent": "^0.1.1",
+        "ms": "^2.1.3",
+        "secure-json-parse": "^2.4.0"
+      },
+      "dependencies": {
+        "ms": {
+          "version": "2.1.3",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+          "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+        }
+      }
+    },
     "@eslint/eslintrc": {
       "version": "0.4.3",
       "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
@@ -12462,6 +12547,24 @@
         "@napi-rs/triples": "^1.0.3"
       }
     },
+    "@opensearch-project/opensearch": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-1.0.2.tgz",
+      "integrity": "sha512-gQ2CjbS7/pJ4A3IBWd+AD0KbCaAnE1Ur9baRxqM1NMH/7A8GQxwtVxx8raUOf4HZExkZZOUYBMzJgWM1OyELyw==",
+      "requires": {
+        "debug": "^4.3.1",
+        "hpagent": "^0.1.1",
+        "ms": "^2.1.3",
+        "secure-json-parse": "^2.4.0"
+      },
+      "dependencies": {
+        "ms": {
+          "version": "2.1.3",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+          "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+        }
+      }
+    },
     "@panva/asn1.js": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/@panva/asn1.js/-/asn1.js-1.0.0.tgz",
@@ -14339,6 +14442,14 @@
         "luxon": "^1.28.0"
       }
     },
+    "cross-fetch": {
+      "version": "3.1.5",
+      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
+      "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==",
+      "requires": {
+        "node-fetch": "2.6.7"
+      }
+    },
     "cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -15782,6 +15893,11 @@
       "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.9.tgz",
       "integrity": "sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw=="
     },
+    "hpagent": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-0.1.2.tgz",
+      "integrity": "sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ=="
+    },
     "htmlparser2": {
       "version": "3.8.3",
       "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz",
@@ -17353,9 +17469,9 @@
       "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA=="
     },
     "node-fetch": {
-      "version": "2.6.5",
+      "version": "2.6.7",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
-      "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
+      "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
       "requires": {
         "whatwg-url": "^5.0.0"
       }
@@ -19021,6 +19137,11 @@
         "node-gyp-build": "^4.2.0"
       }
     },
+    "secure-json-parse": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.4.0.tgz",
+      "integrity": "sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg=="
+    },
     "semver": {
       "version": "6.3.0",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",

package.json

@@ -10,6 +10,8 @@
   },
   "dependencies": {
     "@apollo/client": "^3.4.15",
+    "@elastic/elasticsearch": "^7.16.0",
+    "@opensearch-project/opensearch": "^1.0.2",
     "@prisma/client": "^2.25.0",
     "apollo-server-micro": "^2.21.2",
     "async-retry": "^1.3.1",
@@ -19,6 +21,7 @@
     "bolt11": "^1.3.4",
     "bootstrap": "^4.6.0",
     "clipboard-copy": "^4.0.1",
+    "cross-fetch": "^3.1.5",
     "domino": "^2.1.6",
     "formik": "^2.2.6",
     "graphql": "^15.5.0",

worker/index.js

@@ -5,11 +5,32 @@ const { PrismaClient } = require('@prisma/client')
 const { checkInvoice, checkWithdrawal } = require('./wallet')
 const { repin } = require('./repin')
 const { trust } = require('./trust')
+const { ApolloClient, HttpLink, InMemoryCache } = require('@apollo/client')
+const { indexItem, indexAllItems } = require('./search')
+const fetch = require('cross-fetch')
 
 async function work () {
   const boss = new PgBoss(process.env.DATABASE_URL)
   const models = new PrismaClient()
-  const args = { boss, models }
+  const apollo = new ApolloClient({
+    link: new HttpLink({
+      uri: `${process.env.SELF_URL}/api/graphql`,
+      fetch
+    }),
+    cache: new InMemoryCache(),
+    defaultOptions: {
+      watchQuery: {
+        fetchPolicy: 'network-only',
+        nextFetchPolicy: 'network-only'
+      },
+      query: {
+        fetchPolicy: 'network-only',
+        nextFetchPolicy: 'network-only'
+      }
+    }
+  })
+
+  const args = { boss, models, apollo }
 
   boss.on('error', error => console.error(error))
 
@@ -18,6 +39,8 @@ async function work () {
   await boss.work('checkWithdrawal', checkWithdrawal(args))
   await boss.work('repin-*', repin(args))
   await boss.work('trust', trust(args))
+  await boss.work('indexItem', indexItem(args))
+  await boss.work('indexAllItems', indexAllItems(args))
 
   console.log('working jobs')
 }

worker/search.js

@@ -0,0 +1,97 @@
+const { gql } = require('apollo-server-micro')
+const es = require('@opensearch-project/opensearch')
+
+const search = new es.Client({ node: 'http://localhost:9200' })
+
+const ITEM_SEARCH_FIELDS = gql`
+  fragment ItemSearchFields on Item {
+    id
+    parentId
+    createdAt
+    updatedAt
+    title
+    text
+    url
+    user {
+      name
+    }
+    upvotes
+    sats
+    boost
+    ncomments
+  }`
+
+async function _indexItem (item) {
+  console.log('indexing item', item.id)
+  try {
+    await search.index({
+      id: item.id,
+      index: 'item',
+      version: new Date(item.updatedAt).getTime(),
+      versionType: 'external_gte',
+      body: item
+    })
+  } catch (e) {
+    // ignore version conflict ...
+    if (e?.meta?.statusCode === 409) {
+      console.log('version conflict ignoring', item.id)
+      return
+    }
+    console.log(e)
+    throw e
+  }
+  console.log('done indexing item', item.id)
+}
+
+function indexItem ({ apollo }) {
+  return async function ({ data: { id } }) {
+    // 1. grab item from database
+    // could use apollo to avoid duping logic
+    // when grabbing sats and user name, etc
+    const { data: { item } } = await apollo.query({
+      query: gql`
+        ${ITEM_SEARCH_FIELDS}
+        query Item {
+          item(id: ${id}) {
+            ...ItemSearchFields
+          }
+        }`
+    })
+
+    // 2. index it with external version based on updatedAt
+    await _indexItem(item)
+  }
+}
+
+function indexAllItems ({ apollo }) {
+  return async function () {
+    // cursor over all items in the Item table
+    let items = []; let cursor = null
+    do {
+      // query for items
+      ({ data: { allItems: { items, cursor } } } = await apollo.query({
+        query: gql`
+          ${ITEM_SEARCH_FIELDS}
+          query AllItems($cursor: String) {
+            allItems(cursor: $cursor) {
+              items {
+                ...ItemSearchFields
+              }
+              cursor
+            }
+          }`,
+        variables: { cursor }
+      }))
+
+      // for all items, index them
+      try {
+        items.forEach(_indexItem)
+      } catch (e) {
+        // ignore errors
+        console.log(e)
+      }
+    } while (cursor)
+  }
+}
+
+module.exports = { indexItem, indexAllItems }