ignore subdomains on dupe detection

api/resolvers/item.js

@@ -9,6 +9,7 @@ import {
   MAX_TITLE_LENGTH, ITEM_FILTER_THRESHOLD, DONT_LIKE_THIS_COST
 } from '../../lib/constants'
 import { msatsToSats } from '../../lib/format'
+import { parse } from 'tldts'
 
 async function comments (me, models, id, sort) {
   let orderBy
@@ -469,8 +470,17 @@ export default {
       const urlObj = new URL(ensureProtocol(url))
       let uri = urlObj.hostname + urlObj.pathname
       uri = uri.endsWith('/') ? uri.slice(0, -1) : uri
-      let similar = `(http(s)?://)?${uri}/?`
 
+      const parseResult = parse(urlObj.hostname)
+      if (parseResult?.subdomain?.length) {
+        const { subdomain } = parseResult
+        console.log(subdomain)
+        uri = uri.replace(subdomain, '(%)?')
+      } else {
+        uri = `(%.)?${uri}`
+      }
+
+      let similar = `(http(s)?://)?${uri}/?`
       const whitelist = ['news.ycombinator.com/item', 'bitcointalk.org/index.php']
       const youtube = ['www.youtube.com', 'youtube.com', 'm.youtube.com', 'youtu.be']
       if (whitelist.includes(uri)) {

package-lock.json

@@ -9023,6 +9023,21 @@
       "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz",
       "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA=="
     },
+    "tldts": {
+      "version": "5.7.104",
+      "resolved": "https://registry.npmjs.org/tldts/-/tldts-5.7.104.tgz",
+      "integrity": "sha512-PlziEIVPH/ogbqOhS35K6MOeD09rd9U5g2NHO5n9NZeMC1PGpXgsjQpoJ1KiRnjhZsWDkzN8EoX3xQZuz5ZyFQ==",
+      "requires": {
+        "tldts-core": "^5.7.104"
+      },
+      "dependencies": {
+        "tldts-core": {
+          "version": "5.7.104",
+          "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.7.104.tgz",
+          "integrity": "sha512-8vhSgc2nzPNT0J7XyCqcOtQ6+ySBn+gsPmj5h95YytIZ7L2Xl40paUmj0T6Uko42HegHGQxXieunHIQuABWSmQ=="
+        }
+      }
+    },
     "tldts-core": {
       "version": "5.7.98",
       "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.7.98.tgz",

package.json

@@ -67,6 +67,7 @@
     "sass": "^1.56.0",
     "secp256k1": "^4.0.3",
     "swr": "^1.3.0",
+    "tldts": "^5.7.104",
     "unist-util-visit": "^4.1.1",
     "use-dark-mode": "^2.3.1",
     "uuid": "^8.3.2",