Search filtering by tag (#4312)

Author: rafaeleyng

packages/bsky/src/api/app/bsky/feed/searchPosts.ts

@@ -1,7 +1,12 @@
 import { AtpAgent } from '@atproto/api'
 import { mapDefined } from '@atproto/common'
+import { ServerConfig } from '../../../../config'
 import { AppContext } from '../../../../context'
 import { DataPlaneClient } from '../../../../data-plane'
+import {
+  PostSearchQuery,
+  parsePostSearchQuery,
+} from '../../../../data-plane/server/util'
 import { HydrateCtx, Hydrator } from '../../../../hydration/hydrator'
 import { parseString } from '../../../../hydration/util'
 import { Server } from '../../../../lexicon'
@@ -21,16 +26,20 @@ export default function (server: Server, ctx: AppContext) {
   const searchPosts = createPipeline(
     skeleton,
     hydration,
-    noBlocks,
+    noBlocksOrTagged,
     presentation,
   )
   server.app.bsky.feed.searchPosts({
     auth: ctx.authVerifier.standardOptional,
     handler: async ({ auth, params, req }) => {
-      const viewer = auth.credentials.iss
+      const { viewer, isModService } = ctx.authVerifier.parseCreds(auth)
+
       const labelers = ctx.reqLabelers(req)
       const hydrateCtx = await ctx.hydrator.createContext({ labelers, viewer })
-      const results = await searchPosts({ ...params, hydrateCtx }, ctx)
+      const results = await searchPosts(
+        { ...params, hydrateCtx, isModService },
+        ctx,
+      )
       return {
         encoding: 'application/json',
         body: results,
@@ -42,6 +51,9 @@ export default function (server: Server, ctx: AppContext) {
 
 const skeleton = async (inputs: SkeletonFnInput<Context, Params>) => {
   const { ctx, params } = inputs
+  const parsedQuery = parsePostSearchQuery(params.q, {
+    author: params.author,
+  })
 
   if (ctx.searchAgent) {
     // @NOTE cursors won't change on appview swap
@@ -64,6 +76,7 @@ const skeleton = async (inputs: SkeletonFnInput<Context, Params>) => {
     return {
       posts: res.posts.map(({ uri }) => uri),
       cursor: parseString(res.cursor),
+      parsedQuery,
     }
   }
 
@@ -75,6 +88,7 @@ const skeleton = async (inputs: SkeletonFnInput<Context, Params>) => {
   return {
     posts: res.uris,
     cursor: parseString(res.cursor),
+    parsedQuery,
   }
 }
 
@@ -88,11 +102,30 @@ const hydration = async (
   )
 }
 
-const noBlocks = (inputs: RulesFnInput<Context, Params, Skeleton>) => {
-  const { ctx, skeleton, hydration } = inputs
+const noBlocksOrTagged = (inputs: RulesFnInput<Context, Params, Skeleton>) => {
+  const { ctx, params, skeleton, hydration } = inputs
+  const { parsedQuery } = skeleton
+
   skeleton.posts = skeleton.posts.filter((uri) => {
+    const post = hydration.posts?.get(uri)
+    if (!post) return
+
     const creator = creatorFromUri(uri)
-    return !ctx.views.viewerBlockExists(creator, hydration)
+    const isCuratedSearch = params.sort === 'top'
+    const isPostByViewer = creator === params.hydrateCtx.viewer
+
+    // Cases to always show.
+    if (isPostByViewer) return true
+    if (params.isModService) return true
+
+    // Cases to never show.
+    if (ctx.views.viewerBlockExists(creator, hydration)) return false
+
+    // Cases to conditionally show based on tagging.
+    const tagged = [...ctx.cfg.searchTagsHide].some((t) => post.tags.has(t))
+    if (isCuratedSearch && tagged) return false
+    if (!parsedQuery.author && tagged) return false
+    return true
   })
   return skeleton
 }
@@ -101,9 +134,12 @@ const presentation = (
   inputs: PresentationFnInput<Context, Params, Skeleton>,
 ) => {
   const { ctx, skeleton, hydration } = inputs
-  const posts = mapDefined(skeleton.posts, (uri) =>
-    ctx.views.post(uri, hydration),
-  )
+  const posts = mapDefined(skeleton.posts, (uri) => {
+    const post = hydration.posts?.get(uri)
+    if (!post) return
+
+    return ctx.views.post(uri, hydration)
+  })
   return {
     posts,
     cursor: skeleton.cursor,
@@ -112,16 +148,21 @@ const presentation = (
 }
 
 type Context = {
+  cfg: ServerConfig
   dataplane: DataPlaneClient
   hydrator: Hydrator
   views: Views
   searchAgent?: AtpAgent
 }
 
-type Params = QueryParams & { hydrateCtx: HydrateCtx }
+type Params = QueryParams & {
+  hydrateCtx: HydrateCtx
+  isModService: boolean
+}
 
 type Skeleton = {
   posts: string[]
   hitsTotal?: number
   cursor?: string
+  parsedQuery: PostSearchQuery
 }

packages/bsky/src/auth-verifier.ts

@@ -388,12 +388,17 @@ export class AuthVerifier {
     const canPerformTakedown =
       (creds.credentials.type === 'role' && creds.credentials.admin) ||
       creds.credentials.type === 'mod_service'
+    const isModService =
+      creds.credentials.type === 'mod_service' ||
+      (creds.credentials.type === 'standard' &&
+        this.isModService(creds.credentials.iss))
 
     return {
       viewer,
       includeTakedowns: includeTakedownsAnd3pBlocks,
       include3pBlocks: includeTakedownsAnd3pBlocks,
       canPerformTakedown,
+      isModService,
     }
   }
 }

packages/bsky/src/config.ts

@@ -42,6 +42,7 @@ export interface ServerConfigValues {
   courierHttpVersion?: '1.1' | '2'
   courierIgnoreBadTls?: boolean
   searchUrl?: string
+  searchTagsHide: Set<string>
   suggestionsUrl?: string
   suggestionsApiKey?: string
   topicsUrl?: string
@@ -133,6 +134,7 @@ export class ServerConfig {
       process.env.BSKY_SEARCH_URL ||
       process.env.BSKY_SEARCH_ENDPOINT ||
       undefined
+    const searchTagsHide = new Set(envList(process.env.BSKY_SEARCH_TAGS_HIDE))
     const suggestionsUrl = process.env.BSKY_SUGGESTIONS_URL || undefined
     const suggestionsApiKey = process.env.BSKY_SUGGESTIONS_API_KEY || undefined
     const topicsUrl = process.env.BSKY_TOPICS_URL || undefined
@@ -296,6 +298,7 @@ export class ServerConfig {
       dataplaneHttpVersion,
       dataplaneIgnoreBadTls,
       searchUrl,
+      searchTagsHide,
       suggestionsUrl,
       suggestionsApiKey,
       topicsUrl,
@@ -440,6 +443,10 @@ export class ServerConfig {
     return this.cfg.searchUrl
   }
 
+  get searchTagsHide() {
+    return this.cfg.searchTagsHide
+  }
+
   get suggestionsUrl() {
     return this.cfg.suggestionsUrl
   }
@@ -539,6 +546,7 @@ export class ServerConfig {
   get threadTagsHide() {
     return this.cfg.threadTagsHide
   }
+
   get threadTagsBumpDown() {
     return this.cfg.threadTagsBumpDown
   }

packages/bsky/src/data-plane/server/routes/search.ts

@@ -2,6 +2,7 @@ import { ServiceImpl } from '@connectrpc/connect'
 import { Service } from '../../../proto/bsky_connect'
 import { Database } from '../db'
 import { IndexedAtDidKeyset, TimeCidKeyset, paginate } from '../db/pagination'
+import { parsePostSearchQuery } from '../util'
 
 export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
   // @TODO actor search endpoints still fall back to search service
@@ -35,12 +36,28 @@ export default (db: Database): Partial<ServiceImpl<typeof Service>> => ({
   // @TODO post search endpoint still falls back to search service
   async searchPosts(req) {
     const { term, limit, cursor } = req
+    const { q, author } = parsePostSearchQuery(term)
+
+    let authorDid = author
+    if (author && !author?.startsWith('did:')) {
+      const res = await db.db
+        .selectFrom('actor')
+        .where('handle', '=', author)
+        .selectAll()
+        .executeTakeFirst()
+      authorDid = res?.did
+    }
+
     const { ref } = db.db.dynamic
     let builder = db.db
       .selectFrom('post')
-      .where('post.text', 'like', `%${term}%`)
+      .where('post.text', 'like', `%${q}%`)
       .selectAll()
 
+    if (authorDid) {
+      builder = builder.where('post.creator', '=', authorDid)
+    }
+
     const keyset = new TimeCidKeyset(ref('post.sortAt'), ref('post.cid'))
     builder = paginate(builder, {
       limit,

packages/bsky/src/data-plane/server/util.ts

@@ -153,3 +153,54 @@ export const violatesThreadGate = async (
 
   return true
 }
+
+// @NOTE: This type is not complete with all supported options.
+// Only the ones that we needed to apply custom logic on are currently present.
+export type PostSearchQuery = {
+  q: string
+  author: string | undefined
+}
+
+export const parsePostSearchQuery = (
+  qParam: string,
+  params?: {
+    author?: string
+  },
+): PostSearchQuery => {
+  // Accept individual params, but give preference to options embedded in `q`.
+  let author = params?.author
+
+  const parts: string[] = []
+  let curr = ''
+  let quoted = false
+  for (const c of qParam) {
+    if (c === ' ' && !quoted) {
+      curr.trim() && parts.push(curr)
+      curr = ''
+      continue
+    }
+
+    if (c === '"') {
+      quoted = !quoted
+    }
+    curr += c
+  }
+  curr.trim() && parts.push(curr)
+
+  const qParts: string[] = []
+  for (const p of parts) {
+    const tokens = p.split(':')
+    if (tokens[0] === 'did') {
+      author = p
+    } else if (tokens[0] === 'author' || tokens[0] === 'from') {
+      author = tokens[1]
+    } else {
+      qParts.push(p)
+    }
+  }
+
+  return {
+    q: qParts.join(' '),
+    author,
+  }
+}

packages/bsky/tests/utils.test.ts

@@ -0,0 +1,45 @@
+import {
+  PostSearchQuery,
+  parsePostSearchQuery,
+} from '../src/data-plane/server/util'
+
+describe('parsePostSearchQuery', () => {
+  type TestCase = {
+    input: string
+    output: PostSearchQuery
+  }
+
+  const tests: TestCase[] = [
+    {
+      input: `bluesky `,
+      output: { q: `bluesky`, author: undefined },
+    },
+    {
+      input: ` bluesky  from:esb.lol`,
+      output: { q: `bluesky`, author: `esb.lol` },
+    },
+    {
+      input: `bluesky "from:esb.lol"`,
+      output: { q: `bluesky "from:esb.lol"`, author: undefined },
+    },
+    {
+      input: `bluesky mentions:@esb.lol `,
+      output: { q: `bluesky mentions:@esb.lol`, author: undefined },
+    },
+    {
+      input: `bluesky lang:"en"`,
+      output: { q: `bluesky lang:"en"`, author: undefined },
+    },
+    {
+      input: `bluesky "literal" "from:invalid" did:test:123 `,
+      output: {
+        q: `bluesky "literal" "from:invalid"`,
+        author: `did:test:123`,
+      },
+    },
+  ]
+
+  it.each(tests)(`'$input' -> '$output'`, ({ input, output }) => {
+    expect(parsePostSearchQuery(input)).toEqual(output)
+  })
+})