diff --git a/admin/views/embeddings.php b/admin/views/embeddings.php index 4021c3f..0c06371 100644 --- a/admin/views/embeddings.php +++ b/admin/views/embeddings.php @@ -84,198 +84,44 @@ class="regular-text"> prefix . 'wpvdb_embeddings'; - - // Initialize timing for search performance tracking. $search_start_time = microtime( true ); $search_time_result = 0; $total_vectors_searched = 0; - // Create a database instance instead of using static methods. - $database = new \WPVDB\Database(); - - // Get plugin settings. - $model = \WPVDB\Settings::get_default_model(); - $api_base = \WPVDB\Settings::get_api_base(); - $db_type = $database->get_db_type(); - $has_vector_support = $database->has_native_vector_support() ? 'Yes' : 'No'; + $model = \WPVDB\Settings::get_default_model(); + $api_base = \WPVDB\Settings::get_api_base(); + $api_key = \WPVDB\Settings::get_api_key(); \WPVDB\Logger::debug( 'Performing semantic search for query: ' . $search_query ); - \WPVDB\Logger::debug( 'API key exists: ' . ( ! empty( $api_key ) ? 'Yes' : 'No' ) ); - \WPVDB\Logger::debug( 'Model: ' . $model ); - \WPVDB\Logger::debug( 'API base: ' . $api_base ); if ( $api_key && $model ) { - try { - $embedding_result = \WPVDB\Core::get_embedding( $search_query, $model, $api_base, $api_key ); - - if ( is_wp_error( $embedding_result ) ) { - \WPVDB\Logger::error( 'Error getting embedding: ' . $embedding_result->get_error_message() ); - } else { - \WPVDB\Logger::debug( 'Successfully generated embedding with dimensions: ' . count( $embedding_result ) ); - - $embedding = $embedding_result; - $has_vector = $database->has_native_vector_support(); - \WPVDB\Logger::debug( 'Database has native vector support: ' . ( $has_vector ? 'Yes' : 'No' ) ); - - if ( $has_vector ) { - // Convert the embedding array to JSON. - $embedding_json = wp_json_encode( $embedding ); - - // Use Database class to get the appropriate vector function. - $vector_function = $database->get_vector_from_string_function( $embedding_json ); - \WPVDB\Logger::debug( 'Using vector function: ' . $vector_function ); - - // Get total count of vectors. - $total_vectors_searched = $wpdb->get_var( "SELECT COUNT(*) FROM {$wpdb->prefix}wpvdb_embeddings" ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching - \WPVDB\Logger::debug( 'Total vectors searched: ' . $total_vectors_searched ); - - // Use Database class to get the appropriate distance function with both vectors. - $db_type = $database->get_db_type(); - if ( 'mariadb' === $db_type ) { - $distance_function = "VEC_DISTANCE_COSINE(e.embedding, $vector_function)"; - } else { - $distance_function = "DISTANCE(e.embedding, $vector_function, 'COSINE')"; - } - \WPVDB\Logger::debug( 'Using distance function: ' . $distance_function ); - - // Optimized query that will use the vector index. - // The ORDER BY + LIMIT pattern is what triggers the vector index usage. - // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared - $sql = $wpdb->prepare( - "SELECT e.*, - $distance_function AS distance - FROM $table_name e - WHERE e.model = %s - ORDER BY distance - LIMIT %d", - $model, - 20 // Show top 20 matches. - ); - // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared - - \WPVDB\Logger::debug( 'Executing SQL query: ' . $sql ); - - $search_results = $wpdb->get_results( $sql ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.NotPrepared - - if ( $wpdb->last_error ) { - \WPVDB\Logger::error( 'SQL error: ' . $wpdb->last_error ); - - // Try executing a simpler query to test database connection. - $test_result = $wpdb->get_var( "SELECT COUNT(*) FROM {$wpdb->prefix}wpvdb_embeddings" ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching - - if ( $wpdb->last_error ) { - \WPVDB\Logger::error( 'Even simple query failed: ' . $wpdb->last_error ); - } else { - \WPVDB\Logger::debug( 'Simple query succeeded, embedding count: ' . $test_result ); - - // Try a direct query without the vector function to see if that's the issue. - $basic_results = $wpdb->get_results( "SELECT e.* FROM {$wpdb->prefix}wpvdb_embeddings e LIMIT 20" ); // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching - - if ( $wpdb->last_error ) { - \WPVDB\Logger::error( 'Basic query failed: ' . $wpdb->last_error ); - } else { - \WPVDB\Logger::debug( 'Basic query succeeded, returned ' . count( $basic_results ) . ' results' ); - \WPVDB\Logger::debug( 'Issue is likely with the vector function: ' . $distance_function ); - - // Fall back to PHP-based distance calculation. - \WPVDB\Logger::debug( 'Falling back to PHP-based distance calculation' ); - // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching - $all_rows = $wpdb->get_results( - $wpdb->prepare( "SELECT * FROM {$wpdb->prefix}wpvdb_embeddings WHERE model = %s", $model ), - ARRAY_A - ); - $distances = array(); - - foreach ( $all_rows as $r ) { - $stored_emb = json_decode( $r['embedding'], true ); - if ( ! is_array( $stored_emb ) ) { - continue; - } - $similarity_score = \WPVDB\REST::cosine_distance( $embedding, $stored_emb ); - $r['distance'] = $similarity_score; - $distances[] = $r; - } - - usort( - $distances, - function ( $a, $b ) { - return $a['distance'] <=> $b['distance']; - } - ); - - $search_results = array_slice( $distances, 0, 20 ); - $search_results = json_decode( wp_json_encode( $search_results ) ); // Convert to objects. - - \WPVDB\Logger::debug( 'PHP fallback found ' . count( $search_results ) . ' results' ); - } - } - } else { - \WPVDB\Logger::debug( 'Found ' . count( $search_results ) . ' results' ); - if ( count( $search_results ) > 0 ) { - \WPVDB\Logger::debug( - 'First result distance: ' . - ( isset( $search_results[0]->distance ) ? - $search_results[0]->distance : 'Not set' ) - ); - } - } - } else { - // Fallback: do in PHP. - \WPVDB\Logger::debug( 'Using PHP fallback search' ); - // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching - $all_rows = $wpdb->get_results( - $wpdb->prepare( "SELECT * FROM {$wpdb->prefix}wpvdb_embeddings WHERE model = %s", $model ), - ARRAY_A - ); - $total_vectors_searched = count( $all_rows ); - \WPVDB\Logger::debug( 'Total vectors searched: ' . $total_vectors_searched ); - - $distances = array(); - - foreach ( $all_rows as $r ) { - $stored_emb = json_decode( $r['embedding'], true ); - if ( ! is_array( $stored_emb ) ) { - \WPVDB\Logger::debug( 'Invalid embedding in row: ' . $r['id'] ); - continue; - } - $similarity_score = \WPVDB\REST::cosine_distance( $embedding, $stored_emb ); - $r['distance'] = $similarity_score; - $distances[] = $r; - } - - usort( - $distances, - function ( $a, $b ) { - return $a['distance'] <=> $b['distance']; - } - ); - - $search_results = array_slice( $distances, 0, 20 ); - $search_results = json_decode( wp_json_encode( $search_results ) ); // Convert to objects. - - \WPVDB\Logger::debug( 'PHP fallback found ' . count( $search_results ) . ' results' ); - if ( count( $search_results ) > 0 ) { - \WPVDB\Logger::debug( - 'First result similarity score: ' . - ( isset( $search_results[0]->distance ) ? - $search_results[0]->distance : 'Not set' ) - ); - } - } - - // Use search results instead of regular embeddings. - $embeddings = $search_results; - - // Calculate and record the search time. - $search_time_result = microtime( true ) - $search_start_time; - } - } catch ( \Exception $e ) { - // Handle errors. - \WPVDB\Logger::error( 'Exception: ' . $e->getMessage() ); - echo '

' . esc_html__( 'Error performing semantic search: ', 'wpvdb' ) . esc_html( $e->getMessage() ) . '

'; + $search_response = \WPVDB\Search::query( + array( + 'text' => $search_query, + 'model' => $model, + 'limit' => 20, + // This screen manages the index, so drafts and protected + // posts stay visible here even though the API hides them. + 'respect_visibility' => false, + 'api_base' => $api_base, + 'api_key' => $api_key, + 'output' => OBJECT, + 'explain' => true, + ) + ); + + if ( is_wp_error( $search_response ) ) { + \WPVDB\Logger::error( 'Semantic search failed: ' . $search_response->get_error_message() ); + echo '

' . esc_html__( 'Error performing semantic search: ', 'wpvdb' ) . esc_html( $search_response->get_error_message() ) . '

'; + } else { + $search_results = $search_response['results']; + $embeddings = $search_results; + $total_vectors_searched = (int) $search_response['plan']['total_rows']; } + + $search_time_result = microtime( true ) - $search_start_time; } else { \WPVDB\Logger::error( 'API key or model not configured' ); echo '

' . esc_html__( 'API key or model not configured. Please check your settings.', 'wpvdb' ) . '

'; diff --git a/includes/class-wpvdb-query.php b/includes/class-wpvdb-query.php index e3d9d35..789dd0e 100644 --- a/includes/class-wpvdb-query.php +++ b/includes/class-wpvdb-query.php @@ -13,22 +13,6 @@ * Hooks vector search into WordPress queries. */ class Query { - /** - * Database handler - * - * @var Database - */ - private static $database; - - /** - * Initialize the database instance - */ - private static function init_database() { - if ( null === self::$database ) { - self::$database = new Database(); - } - } - /** * Hook into 'pre_get_posts' or a similar filter to do custom vector searching if requested. */ @@ -49,9 +33,6 @@ public static function init() { * @return void */ public static function maybe_vector_search( $query ) { - // Initialize database. - self::init_database(); - // Only run in front-end or REST contexts, and only if vdb_vector_query is set. if ( is_admin() && ! wp_doing_ajax() ) { return; @@ -66,11 +47,6 @@ public static function maybe_vector_search( $query ) { Logger::debug( 'maybe_vector_search triggered with query: ' . $vdb_query ); - // For simplicity, embed and do a fallback search. Then get the doc_ids, presumably post_id was stored as doc_id. - global $wpdb; - $table_name = $wpdb->prefix . 'wpvdb_embeddings'; - - // We'll do a direct call to the REST method or replicate logic from REST::handle_query. $api_key = apply_filters( 'wpvdb_default_api_key', '' ); if ( ! $api_key ) { $api_key = Settings::get_api_key(); @@ -96,106 +72,43 @@ public static function maybe_vector_search( $query ) { Logger::debug( 'Using API base: ' . $api_base ); try { - $embedding_result = Core::get_embedding( $vdb_query, $model, $api_base, $api_key ); - if ( is_wp_error( $embedding_result ) ) { - Logger::error( 'Error generating embedding: ' . $embedding_result->get_error_message() ); - return; // skip. + // posts_per_page of -1 or 0 has no bounded meaning here, so use the default page size. + $limit = (int) $query->get( 'posts_per_page' ); + $limit = $limit > 0 ? $limit : 10; + + // Set an appropriate similarity threshold - we discovered this is critical for performance + // Lower values (0.2-0.3) are more strict but faster, higher values (0.4-0.6) give more results. + $similarity_threshold = apply_filters( 'wpvdb_similarity_threshold', 0.35 ); + + $search = Search::query( + array( + 'text' => $vdb_query, + 'model' => $model, + // Over-fetch: several chunks can resolve to the same post and + // collapse when doc_ids are deduped below. + 'limit' => $limit * 3, + 'distance_threshold' => $similarity_threshold, + // WP_Query re-gates status and capabilities downstream. + 'respect_visibility' => false, + 'api_base' => $api_base, + 'api_key' => $api_key, + ) + ); + + if ( is_wp_error( $search ) ) { + Logger::error( 'Vector search failed: ' . $search->get_error_message() ); + return; } - Logger::debug( 'Embedding generated successfully, dimensions: ' . count( $embedding_result ) ); - - $embedding = $embedding_result; - $has_vector = self::$database->has_native_vector_support(); - Logger::debug( 'Vector support detected: ' . ( $has_vector ? 'Yes' : 'No' ) ); - - $limit = $query->get( 'posts_per_page' ); - $limit = $limit ? $limit : 10; - Logger::debug( 'Posts per page limit: ' . $limit ); - - $doc_ids = array(); - - if ( $has_vector ) { - try { - // Convert the embedding array to JSON. - $embedding_json = wp_json_encode( $embedding ); - - // Use Database class to get the appropriate vector function. - $vector_function = self::$database->get_vector_from_string_function( $embedding_json ); - Logger::debug( 'Using vector function: ' . $vector_function ); - - // Use Database class to get the appropriate distance function. - $distance_function = self::$database->get_vector_distance_function( 'embedding', $vector_function, 'cosine' ); - Logger::debug( 'Using distance function: ' . $distance_function ); - - // Set an appropriate similarity threshold - we discovered this is critical for performance - // Lower values (0.2-0.3) are more strict but faster, higher values (0.4-0.6) give more results. - $similarity_threshold = apply_filters( 'wpvdb_similarity_threshold', 0.35 ); - - // Optimized query that uses the vector index with a distance threshold. - // The threshold + ORDER BY + LIMIT pattern is what maximizes vector index usage. - $sql = $wpdb->prepare( - " - SELECT doc_id, - $distance_function AS distance - FROM $table_name - WHERE $distance_function < %f - AND model = %s - ORDER BY distance - LIMIT %d - ", - $similarity_threshold, - $model, - $limit * 3 // fetch more candidates than needed. - ); - - Logger::debug( 'Vector search SQL: ' . $sql ); - - $rows = $wpdb->get_results( $sql, ARRAY_A ); - - if ( $wpdb->last_error ) { - Logger::error( 'Database error in vector search: ' . $wpdb->last_error ); - } - - if ( $rows ) { - Logger::debug( 'Found ' . count( $rows ) . ' results from vector search' ); - foreach ( $rows as $r ) { - $doc_ids[] = (int) $r['doc_id']; - Logger::debug( 'Added doc_id: ' . $r['doc_id'] . ' with distance: ' . $r['distance'] ); - } - } else { - Logger::debug( 'No results found from vector search' ); - } - } catch ( \Exception $e ) { - Logger::error( 'Exception in vector search: ' . $e->getMessage() ); - } - } else { - Logger::debug( 'No vector support, using PHP fallback search' ); - // Fallback: do in PHP. - $all_rows = $wpdb->get_results( - $wpdb->prepare( "SELECT doc_id, embedding FROM $table_name WHERE model = %s", $model ), - ARRAY_A - ); - $distances = array(); - foreach ( $all_rows as $r ) { - $stored_emb = json_decode( $r['embedding'], true ); - if ( ! is_array( $stored_emb ) ) { - continue; - } - $d = REST::cosine_distance( $embedding, $stored_emb ); - $distances[] = array( - 'doc_id' => (int) $r['doc_id'], - 'distance' => $d, - ); - } - usort( - $distances, - function ( $a, $b ) { - return $a['distance'] <=> $b['distance']; - } - ); - $distances = array_slice( $distances, 0, $limit * 3 ); - $doc_ids = wp_list_pluck( $distances, 'doc_id' ); - } + $doc_ids = array_map( 'intval', wp_list_pluck( $search['results'], 'doc_id' ) ); + + Logger::debug( + 'Vector search completed', + array( + 'strategy' => $search['plan']['strategy'], + 'matches' => count( $doc_ids ), + ) + ); if ( empty( $doc_ids ) ) { // No matches, so force query to return no posts. diff --git a/includes/class-wpvdb-rest.php b/includes/class-wpvdb-rest.php index b853107..0110067 100644 --- a/includes/class-wpvdb-rest.php +++ b/includes/class-wpvdb-rest.php @@ -23,6 +23,16 @@ */ class REST { + /** + * Candidate multiplier applied to the requested result count. + * + * The visibility gate runs in SQL, so reading only `limit` rows lets + * filtered-out rows shrink the response below what the client asked for. + * + * @var int + */ + const QUERY_OVER_FETCH = 3; + /** * Database handler instance * @@ -566,9 +576,6 @@ public static function handle_query( \WP_REST_Request $request ) { $start_time = Logger::start_timer( 'query_processing' ); try { - global $wpdb; - $table_name = $wpdb->prefix . 'wpvdb_embeddings'; - Logger::debug( 'Processing query request', array( @@ -578,19 +585,11 @@ public static function handle_query( \WP_REST_Request $request ) { ) ); - if ( $has_provided_vector ) { - $embedding = $normalized_vector; - } else { - Logger::debug( - 'Using configuration', - array( - 'model' => $model, - 'provider' => $provider, - ) - ); - - // Get API key from settings based on provider. + // Pre-flighted here rather than in Search so the REST-specific + // error codes existing clients match on are preserved. + if ( ! $has_provided_vector ) { $api_key = Settings::get_api_key_for_provider( $provider ); + if ( empty( $api_key ) ) { Logger::error( 'API key not configured', array( 'provider' => $provider ) ); return new \WP_Error( 'missing_api_key', __( 'API key not configured for the selected provider', 'wpvdb' ), array( 'status' => 400 ) ); @@ -600,245 +599,49 @@ public static function handle_query( \WP_REST_Request $request ) { Logger::error( 'API base URL not configured', array( 'provider' => $provider ) ); return new \WP_Error( 'missing_api_base', __( 'API base URL not configured for the selected provider', 'wpvdb' ), array( 'status' => 400 ) ); } + } - Logger::debug( - 'Generating embedding', - array( - 'model' => $model, - 'text_length' => strlen( $text ), - ) - ); + $search = Search::query( + array( + 'text' => $has_provided_vector ? '' : $text, + 'vector' => $has_provided_vector ? $normalized_vector : null, + 'model' => $model, + 'limit' => $limit, + 'over_fetch' => self::QUERY_OVER_FETCH, + 'respect_visibility' => true, + 'provider' => $provider, + 'api_base' => $api_base, + ) + ); - $embed_start = $debug ? microtime( true ) : 0.0; - $embedding = Core::get_embedding( $text, $model, $api_base, $api_key ); - if ( $debug ) { - $timing['embed_ms'] = (int) round( ( microtime( true ) - $embed_start ) * 1000 ); - } - if ( is_wp_error( $embedding ) ) { - Logger::error( - 'Failed to generate embedding', - array( - 'error' => $embedding->get_error_message(), - 'model' => $model, - ) - ); - return $embedding; - } + if ( is_wp_error( $search ) ) { + return $search; } - Logger::debug( 'Embedding generated successfully', array( 'dimensions' => count( $embedding ) ) ); + $results = $search['results']; + $plan = $search['plan']; + $has_vector = $plan['has_vector_support']; - // Now we have an embedding array of floats. If we have native vector support, use it. Otherwise fallback. - $probe_start = $debug ? microtime( true ) : 0.0; - $has_vector = self::$database->has_native_vector_support(); if ( $debug ) { - $timing['vector_probe_ms'] = (int) round( ( microtime( true ) - $probe_start ) * 1000 ); + $timing['embed_ms'] = $plan['timings_ms']['embed']; + $timing['vector_probe_ms'] = $plan['timings_ms']['vector_probe']; + $timing['db_ms'] = $plan['timings_ms']['db']; } - Logger::debug( 'Database vector support status', array( 'has_vector' => $has_vector ) ); - $results = array(); - - if ( $has_vector ) { - try { - // Convert the embedding array to JSON and validate. - $embedding_json = wp_json_encode( $embedding ); - if ( false === $embedding_json ) { - return new \WP_Error( 'encoding_error', __( 'Failed to encode embedding data', 'wpvdb' ), array( 'status' => 500 ) ); - } - - // Use Database class to get safe vector SQL components. - $db_type = self::$database->get_db_type(); - $vector_function = ''; - $distance_function = ''; - - // Build safe SQL based on database type. MariaDB 11.7+ uses - // VEC_FromText to parse a JSON array; MySQL 9 uses its own - // ingest function (VECTOR_FROM_JSON here is a placeholder). - if ( 'mariadb' === $db_type ) { - $vector_function = "VEC_FromText('" . esc_sql( $embedding_json ) . "')"; - $distance_function = 'VEC_DISTANCE_COSINE(embedding, ' . $vector_function . ')'; - } elseif ( 'mysql' === $db_type ) { - $vector_function = "VECTOR_FROM_JSON('" . esc_sql( $embedding_json ) . "')"; - $distance_function = 'COSINE_DISTANCE(embedding, ' . $vector_function . ')'; - } else { - return new \WP_Error( 'db_error', __( 'Unsupported database type for vector operations', 'wpvdb' ), array( 'status' => 500 ) ); - } - - Logger::debug( - 'Vector SQL components', - array( - 'vector_function' => substr( $vector_function, 0, 30 ) . '...', - 'distance_function' => substr( $distance_function, 0, 50 ) . '...', - 'db_type' => $db_type, - ) - ); - // Scope by model (no cross-model rows) and exclude protected / - // non-public post rows at query time (arbitrary docs pass). - $sql = $wpdb->prepare( - "SELECT e.id, e.doc_id, e.chunk_id, e.chunk_content, e.summary, - {$distance_function} as distance - FROM {$table_name} e - LEFT JOIN {$wpdb->posts} p ON p.ID = e.doc_id - WHERE e.model = %s - AND ( p.ID IS NULL OR ( p.post_status = 'publish' AND p.post_password = '' ) ) - ORDER BY distance - LIMIT %d", - $model, - $limit - ); - - Logger::debug( 'Executing vector query', array( 'limit' => $limit ) ); - - $db_start = $debug ? microtime( true ) : 0.0; - $results = $wpdb->get_results( $sql, ARRAY_A ); - if ( $debug ) { - $timing['db_ms'] = (int) round( ( microtime( true ) - $db_start ) * 1000 ); - } - - if ( $wpdb->last_error ) { - Logger::error( - 'Vector query database error', - array( - 'error' => $wpdb->last_error, - 'sql' => substr( $sql, 0, 200 ) . '...', - ) - ); - return new \WP_Error( 'db_error', $wpdb->last_error, array( 'status' => 500 ) ); - } - - Logger::info( - 'Vector query completed', - array( - 'results_count' => count( $results ), - 'has_vector' => true, - ) - ); - } catch ( \Exception $e ) { - Logger::log_exception( $e, 'Vector query exception' ); - return new \WP_Error( 'query_error', $e->getMessage(), array( 'status' => 500 ) ); - } - } else { - // Fallback to PHP with pagination and memory optimization. - Logger::warning( 'Using PHP fallback for similarity search - performance may be slower' ); - $fallback_start = microtime( true ); - - // Use pagination to avoid loading all rows at once. - $page_size = 1000; - $offset = 0; - $distances = array(); - $total_processed = 0; - - while ( true ) { - // Get a batch of rows with LIMIT and OFFSET. Same visibility - // filter as the native path (exclude protected/non-public posts). - $batch_query = $wpdb->prepare( - "SELECT e.id, e.doc_id, e.chunk_id, e.chunk_content, e.summary, e.embedding - FROM {$table_name} e - LEFT JOIN {$wpdb->posts} p ON p.ID = e.doc_id - WHERE e.model = %s - AND ( p.ID IS NULL OR ( p.post_status = 'publish' AND p.post_password = '' ) ) - LIMIT %d OFFSET %d", - $model, - $page_size, - $offset - ); - - $batch_rows = $wpdb->get_results( $batch_query, ARRAY_A ); - - if ( $wpdb->last_error ) { - Logger::error( - 'PHP fallback database error', - array( - 'error' => $wpdb->last_error, - 'offset' => $offset, - ) - ); - return new \WP_Error( 'db_error', $wpdb->last_error, array( 'status' => 500 ) ); - } - - // Break if no more rows. - if ( empty( $batch_rows ) ) { - break; - } - - // Process this batch. - foreach ( $batch_rows as $row ) { - try { - $vector = json_decode( $row['embedding'], true ); - if ( ! is_array( $vector ) ) { - continue; // Skip invalid embeddings. - } - - $distance = self::cosine_distance( $embedding, $vector ); - - // Add distance to the row. - $row['distance'] = $distance; - $distances[] = $row; - ++$total_processed; - - // Memory management: if we have way more than needed, - // sort and trim to prevent memory issues. - if ( count( $distances ) > ( $limit * 10 ) ) { - usort( - $distances, - function ( $a, $b ) { - return $a['distance'] <=> $b['distance']; - } - ); - $distances = array_slice( $distances, 0, $limit * 2 ); - } - } catch ( \Exception $e ) { - // Skip rows that cause errors. - Logger::warning( - 'Error processing embedding row in fallback', - array( - 'row_id' => $row['id'], - 'error' => $e->getMessage(), - ) - ); - } - } - - $offset += $page_size; - - // Safety break to prevent infinite loops. - if ( $total_processed > 50000 ) { - Logger::warning( 'Fallback processing limit reached', array( 'processed' => $total_processed ) ); - break; - } - } - - // Final sort and limit. - usort( - $distances, - function ( $a, $b ) { - return $a['distance'] <=> $b['distance']; - } - ); - - // Limit results. - $results = array_slice( $distances, 0, $limit ); - - $fallback_duration = microtime( true ) - $fallback_start; - if ( $debug ) { - $timing['db_ms'] = (int) round( $fallback_duration * 1000 ); - } - Logger::log_performance( - 'php_fallback_similarity_search', - $fallback_duration, - array( - 'total_processed' => $total_processed, - 'results_returned' => count( $results ), - ) - ); - } + Logger::info( + 'Query completed', + array( + 'results_count' => count( $results ), + 'strategy' => $plan['strategy'], + ) + ); // Add debug info. $results = array_map( - function ( $row ) { + function ( $row ) use ( $plan ) { $row['debug_info'] = array( - 'database_type' => self::$database->get_db_type(), - 'has_vector_support' => self::$database->has_native_vector_support() ? 'yes' : 'no', + 'database_type' => $plan['db_type'], + 'has_vector_support' => $plan['has_vector_support'] ? 'yes' : 'no', ); return $row; }, @@ -1078,7 +881,6 @@ public static function insert_embedding_row( $doc_id, $chunk_id, $chunk_content, // Use the Database class to determine the vector function to use. $vector_function = self::$database->get_vector_from_string_function( $embedding_json ); - Logger::debug( 'Vector function: ' . $vector_function ); // For MySQL, the prepare statement handles the quoting properly // For MariaDB, we need to make sure the vector function is inserted as-is. @@ -1088,8 +890,8 @@ public static function insert_embedding_row( $doc_id, $chunk_id, $chunk_content, // Cache::get_relevant_embeddings() and Maintenance compare against. $sql = $wpdb->prepare( "INSERT INTO $table_name - (doc_id, chunk_id, chunk_content, summary, embedding, model, doc_type, chunk_index, embedding_date) - VALUES (%d, %s, %s, %s, $vector_function, %s, %s, %d, NOW())", + (doc_id, chunk_id, chunk_content, summary, embedding, model, doc_type, chunk_index, embedding_date) + VALUES (%d, %s, %s, %s, $vector_function, %s, %s, %d, NOW())", $doc_id, $chunk_id, $chunk_content, diff --git a/includes/class-wpvdb-search.php b/includes/class-wpvdb-search.php new file mode 100644 index 0000000..5ab543c --- /dev/null +++ b/includes/class-wpvdb-search.php @@ -0,0 +1,489 @@ + '', + 'vector' => null, + 'model' => '', + 'limit' => 10, + 'over_fetch' => 1, + 'distance_threshold' => null, + 'respect_visibility' => true, + 'provider' => '', + 'api_base' => '', + 'api_key' => '', + 'output' => ARRAY_A, + 'explain' => false, + ); + } + + /** + * Run a similarity search. + * + * @param array $args { + * Search arguments. + * + * @type string $text Text to embed and search with. Ignored when `vector` is set. + * @type float[] $vector Pre-computed query vector. + * @type string $model Embedding model to scope results to. Defaults to the configured model. + * @type int $limit Number of rows to return. + * @type int $over_fetch Multiplier applied to `limit` when reading candidates. + * @type float|null $distance_threshold Discard rows at or above this distance. + * @type bool $respect_visibility Exclude non-public and password-protected posts. + * @type string $provider Provider override used to resolve credentials. + * @type string $api_base API base override. + * @type string $api_key API key override. + * @type string $output ARRAY_A or OBJECT. + * @type bool $explain Include corpus size in the returned plan. + * } + * @return array|\WP_Error { + * @type array $results Result rows ordered by ascending distance. + * @type array $plan How the search was executed. + * } + */ + public static function query( array $args ) { + $args = wp_parse_args( $args, self::default_args() ); + + $args['model'] = $args['model'] ? $args['model'] : Settings::get_default_model(); + $args['limit'] = max( 1, (int) $args['limit'] ); + $args['over_fetch'] = max( 1, (int) $args['over_fetch'] ); + + $plan = array( + 'strategy' => '', + 'model' => $args['model'], + 'db_type' => self::db()->get_db_type(), + 'has_vector_support' => null, + 'candidates' => null, + 'rows_scanned' => null, + 'total_rows' => null, + 'timings_ms' => array( + 'embed' => 0, + 'vector_probe' => 0, + 'db' => 0, + ), + ); + + $embedding = $args['vector']; + if ( ! is_array( $embedding ) ) { + if ( '' === trim( (string) $args['text'] ) ) { + return new \WP_Error( + 'wpvdb_search_no_input', + __( 'A query text or a query vector is required.', 'wpvdb' ), + array( 'status' => 400 ) + ); + } + + $started = microtime( true ); + $embedding = self::resolve_embedding( $args ); + + $plan['timings_ms']['embed'] = (int) round( ( microtime( true ) - $started ) * 1000 ); + + if ( is_wp_error( $embedding ) ) { + return $embedding; + } + } + + $started = microtime( true ); + $has_vector = self::db()->has_native_vector_support(); + $plan['timings_ms']['vector_probe'] = (int) round( ( microtime( true ) - $started ) * 1000 ); + $plan['has_vector_support'] = $has_vector; + $plan['strategy'] = $has_vector ? 'native' : 'php_fallback'; + + $fetch = $args['limit'] * $args['over_fetch']; + + $started = microtime( true ); + $rows = $has_vector + ? self::run_native( $embedding, $fetch, $args ) + : self::run_php_fallback( $embedding, $fetch, $args, $plan ); + $plan['timings_ms']['db'] = (int) round( ( microtime( true ) - $started ) * 1000 ); + + if ( is_wp_error( $rows ) ) { + return $rows; + } + + $rows = array_slice( $rows, 0, $args['limit'] ); + + if ( $args['explain'] ) { + $plan['total_rows'] = self::count_rows( $args['model'] ); + } + + if ( OBJECT === $args['output'] ) { + $rows = array_map( + function ( $row ) { + return (object) $row; + }, + $rows + ); + } + + return array( + 'results' => $rows, + 'plan' => $plan, + ); + } + + /** + * Build the JOIN and WHERE fragments applied to a search. + * + * Extension point for callers that need to constrain the candidate set. + * Filters must return the same array shape; `params` values are bound in + * the order the corresponding placeholders appear in `where`. + * + * @param array $args Normalized search arguments. + * @return array { + * @type string $join SQL fragment appended after the table alias `e`. + * @type string[] $where Conditions combined with AND. + * @type array $params Values bound to placeholders in `where`. + * } + */ + public static function build_clauses( array $args ) { + global $wpdb; + + $clauses = array( + 'join' => '', + 'where' => array( 'e.model = %s' ), + 'params' => array( $args['model'] ), + ); + + if ( ! empty( $args['respect_visibility'] ) ) { + $clauses['join'] .= " LEFT JOIN {$wpdb->posts} p ON p.ID = e.doc_id"; + $clauses['where'][] = "( p.ID IS NULL OR ( p.post_status = 'publish' AND p.post_password = '' ) )"; + } + + return apply_filters( 'wpvdb_search_clauses', $clauses, $args ); + } + + /** + * Run the search using native vector SQL. + * + * @param array $embedding Query vector. + * @param int $fetch Rows to read. + * @param array $args Normalized search arguments. + * @return array|\WP_Error + */ + private static function run_native( array $embedding, $fetch, array $args ) { + global $wpdb; + + $embedding_json = wp_json_encode( $embedding ); + if ( false === $embedding_json ) { + return new \WP_Error( + 'wpvdb_search_encoding_error', + __( 'Failed to encode the query vector.', 'wpvdb' ), + array( 'status' => 500 ) + ); + } + + $vector_sql = self::db()->get_vector_from_string_function( $embedding_json ); + $distance = self::db()->get_vector_distance_function( 'e.embedding', $vector_sql, 'cosine' ); + + // The distance fragment is interpolated into a prepare() format string, + // so any literal percent inside the serialized vector must be escaped. + $distance_format = str_replace( '%', '%%', $distance ); + + $clauses = self::build_clauses( $args ); + $params = $clauses['params']; + + if ( null !== $args['distance_threshold'] ) { + $clauses['where'][] = $distance_format . ' < %f'; + $params[] = (float) $args['distance_threshold']; + } + + $columns = self::column_list( 'e.' ); + $table = self::table(); + $where = implode( ' AND ', $clauses['where'] ); + $params[] = (int) $fetch; + + // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared + $sql = $wpdb->prepare( + "SELECT {$columns}, {$distance_format} AS distance + FROM {$table} e{$clauses['join']} + WHERE {$where} + ORDER BY distance + LIMIT %d", + $params + ); + // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared + + Logger::debug( 'Running native vector search', array( 'fetch' => $fetch ) ); + + // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.NotPrepared + $rows = $wpdb->get_results( $sql, ARRAY_A ); + + if ( $wpdb->last_error ) { + Logger::error( + 'Vector search database error', + array( + 'error' => $wpdb->last_error, + 'sql' => substr( $sql, 0, 200 ), + ) + ); + + return new \WP_Error( 'wpvdb_search_db_error', $wpdb->last_error, array( 'status' => 500 ) ); + } + + return is_array( $rows ) ? $rows : array(); + } + + /** + * Score candidates in PHP for databases without native vector support. + * + * Reads in batches and trims the working set so a large corpus does not + * have to be held in memory at once. + * + * @param array $embedding Query vector. + * @param int $fetch Rows to return. + * @param array $args Normalized search arguments. + * @param array $plan Plan array, updated by reference with scan counters. + * @return array|\WP_Error + */ + private static function run_php_fallback( array $embedding, $fetch, array $args, array &$plan ) { + global $wpdb; + + Logger::warning( 'Using PHP fallback for similarity search - performance may be slower' ); + + $fallback_start = microtime( true ); + $clauses = self::build_clauses( $args ); + $columns = self::column_list( 'e.' ); + $table = self::table(); + $where = implode( ' AND ', $clauses['where'] ); + $distances = array(); + $scanned = 0; + $offset = 0; + + while ( true ) { + $params = $clauses['params']; + $params[] = self::FALLBACK_PAGE_SIZE; + $params[] = $offset; + + // Placeholder count is dynamic because build_clauses() is filterable. + // phpcs:disable WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber + $sql = $wpdb->prepare( + "SELECT {$columns}, e.embedding + FROM {$table} e{$clauses['join']} + WHERE {$where} + LIMIT %d OFFSET %d", + $params + ); + // phpcs:enable WordPress.DB.PreparedSQL.InterpolatedNotPrepared, WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber + + // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.NotPrepared + $batch = $wpdb->get_results( $sql, ARRAY_A ); + + if ( $wpdb->last_error ) { + Logger::error( + 'PHP fallback database error', + array( + 'error' => $wpdb->last_error, + 'offset' => $offset, + ) + ); + + return new \WP_Error( 'wpvdb_search_db_error', $wpdb->last_error, array( 'status' => 500 ) ); + } + + if ( empty( $batch ) ) { + break; + } + + foreach ( $batch as $row ) { + $stored = json_decode( $row['embedding'], true ); + if ( ! is_array( $stored ) ) { + continue; + } + + unset( $row['embedding'] ); + $row['distance'] = REST::cosine_distance( $embedding, $stored ); + ++$scanned; + + if ( null !== $args['distance_threshold'] && $row['distance'] >= (float) $args['distance_threshold'] ) { + continue; + } + + $distances[] = $row; + + if ( count( $distances ) > ( $fetch * 10 ) ) { + $distances = self::sort_by_distance( $distances ); + $distances = array_slice( $distances, 0, $fetch * 2 ); + } + } + + $offset += self::FALLBACK_PAGE_SIZE; + + if ( $scanned > self::FALLBACK_MAX_ROWS ) { + Logger::warning( 'Fallback processing limit reached', array( 'processed' => $scanned ) ); + break; + } + } + + $plan['rows_scanned'] = $scanned; + $results = array_slice( self::sort_by_distance( $distances ), 0, $fetch ); + + Logger::log_performance( + 'php_fallback_similarity_search', + microtime( true ) - $fallback_start, + array( + 'total_processed' => $scanned, + 'results_returned' => count( $results ), + ) + ); + + return $results; + } + + /** + * Turn the query text into a vector using the resolved provider credentials. + * + * @param array $args Normalized search arguments. + * @return array|\WP_Error + */ + private static function resolve_embedding( array $args ) { + $provider = $args['provider'] ? $args['provider'] : Settings::get_active_provider(); + $api_base = $args['api_base'] ? $args['api_base'] : Settings::get_api_base_for_provider( $provider ); + $api_key = $args['api_key'] ? $args['api_key'] : Settings::get_api_key_for_provider( $provider ); + + if ( empty( $api_key ) ) { + return new \WP_Error( + 'wpvdb_search_missing_api_key', + __( 'API key not configured for the selected provider.', 'wpvdb' ), + array( 'status' => 400 ) + ); + } + + if ( empty( $api_base ) ) { + return new \WP_Error( + 'wpvdb_search_missing_api_base', + __( 'API base URL not configured for the selected provider.', 'wpvdb' ), + array( 'status' => 400 ) + ); + } + + return Core::get_embedding( $args['text'], $args['model'], $api_base, $api_key ); + } + + /** + * Count stored rows for a model. + * + * @param string $model Model name. + * @return int + */ + private static function count_rows( $model ) { + global $wpdb; + + $table = self::table(); + + // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching, WordPress.DB.PreparedSQL.InterpolatedNotPrepared + return (int) $wpdb->get_var( $wpdb->prepare( "SELECT COUNT(*) FROM {$table} WHERE model = %s", $model ) ); + } + + /** + * Sort rows by ascending distance. + * + * @param array $rows Rows carrying a `distance` key. + * @return array + */ + private static function sort_by_distance( array $rows ) { + usort( + $rows, + function ( $a, $b ) { + return $a['distance'] <=> $b['distance']; + } + ); + + return $rows; + } + + /** + * Comma-separated result column list. + * + * @param string $prefix Column prefix, e.g. "e.". + * @return string + */ + private static function column_list( $prefix = '' ) { + return $prefix . implode( ', ' . $prefix, self::RESULT_COLUMNS ); + } + + /** + * Fully qualified embeddings table name. + * + * @return string + */ + private static function table() { + global $wpdb; + + return $wpdb->prefix . 'wpvdb_embeddings'; + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 6092bc6..451407a 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -554,6 +554,7 @@ class wpdb { public $last_result = []; public $num_rows = 0; public $prefix = 'wp_'; + public $posts = 'wp_posts'; public $dbname = 'test_db'; public function get_var( $query = null, $x = 0, $y = 0 ) { @@ -621,6 +622,7 @@ public function suppress_errors( $suppress = true ) { require_once dirname( __DIR__ ) . '/includes/class-wpvdb-indexability.php'; require_once dirname( __DIR__ ) . '/includes/class-wpvdb-core.php'; require_once dirname( __DIR__ ) . '/includes/class-wpvdb-database.php'; +require_once dirname( __DIR__ ) . '/includes/class-wpvdb-search.php'; require_once dirname( __DIR__ ) . '/includes/class-wpvdb-rest.php'; require_once dirname( __DIR__ ) . '/includes/class-wpvdb-embedding-enqueuer.php'; require_once dirname( __DIR__ ) . '/includes/class-wpvdb-queue.php'; diff --git a/tests/unit/SearchTest.php b/tests/unit/SearchTest.php new file mode 100644 index 0000000..76c284e --- /dev/null +++ b/tests/unit/SearchTest.php @@ -0,0 +1,135 @@ +assertNotContains( 'embedding', Search::RESULT_COLUMNS ); + $this->assertContains( 'doc_id', Search::RESULT_COLUMNS ); + $this->assertContains( 'chunk_content', Search::RESULT_COLUMNS ); + $this->assertContains( 'summary', Search::RESULT_COLUMNS ); + } + + /** + * Defaults gate results to publicly visible posts. + */ + public function test_visibility_is_respected_by_default() { + $defaults = Search::default_args(); + + $this->assertTrue( $defaults['respect_visibility'] ); + $this->assertNull( $defaults['distance_threshold'] ); + $this->assertSame( 1, $defaults['over_fetch'] ); + } + + /** + * Every search is scoped to a single model. + */ + public function test_clauses_always_scope_to_the_model() { + $clauses = Search::build_clauses( $this->args( array( 'model' => 'text-embedding-3-small' ) ) ); + + $this->assertContains( 'e.model = %s', $clauses['where'] ); + $this->assertSame( array( 'text-embedding-3-small' ), $clauses['params'] ); + } + + /** + * The visibility gate adds a posts join and a status condition. + */ + public function test_clauses_join_posts_when_visibility_is_respected() { + $clauses = Search::build_clauses( + $this->args( + array( + 'model' => 'm', + 'respect_visibility' => true, + ) + ) + ); + + $this->assertStringContainsString( 'LEFT JOIN', $clauses['join'] ); + $this->assertStringContainsString( 'p.ID = e.doc_id', $clauses['join'] ); + + $where = implode( ' AND ', $clauses['where'] ); + $this->assertStringContainsString( "p.post_status = 'publish'", $where ); + $this->assertStringContainsString( "p.post_password = ''", $where ); + + // Rows for non-post documents survive the gate. + $this->assertStringContainsString( 'p.ID IS NULL', $where ); + } + + /** + * Opting out of the gate leaves the query unjoined. + */ + public function test_clauses_skip_the_join_when_visibility_is_not_respected() { + $clauses = Search::build_clauses( + $this->args( + array( + 'model' => 'm', + 'respect_visibility' => false, + ) + ) + ); + + $this->assertSame( '', $clauses['join'] ); + $this->assertSame( array( 'e.model = %s' ), $clauses['where'] ); + } + + /** + * Clauses are filterable so callers can constrain the candidate set. + */ + public function test_clauses_are_filterable() { + add_filter( + 'wpvdb_search_clauses', + function ( $clauses ) { + $clauses['where'][] = 'e.doc_type = %s'; + $clauses['params'][] = 'product'; + return $clauses; + } + ); + + $clauses = Search::build_clauses( + $this->args( + array( + 'model' => 'm', + 'respect_visibility' => false, + ) + ) + ); + + $this->assertSame( array( 'e.model = %s', 'e.doc_type = %s' ), $clauses['where'] ); + $this->assertSame( array( 'm', 'product' ), $clauses['params'] ); + } + + /** + * A search with neither text nor a vector is rejected before any API call. + */ + public function test_query_requires_text_or_a_vector() { + $result = Search::query( array( 'text' => ' ' ) ); + + $this->assertInstanceOf( \WP_Error::class, $result ); + $this->assertSame( 'wpvdb_search_no_input', $result->get_error_code() ); + } +} diff --git a/wpvdb.php b/wpvdb.php index cd07d33..c416061 100644 --- a/wpvdb.php +++ b/wpvdb.php @@ -68,6 +68,7 @@ require_once WPVDB_PLUGIN_DIR . 'includes/class-wpvdb-models.php'; require_once WPVDB_PLUGIN_DIR . 'includes/class-wpvdb-providers.php'; require_once WPVDB_PLUGIN_DIR . 'includes/class-wpvdb-core.php'; +require_once WPVDB_PLUGIN_DIR . 'includes/class-wpvdb-search.php'; require_once WPVDB_PLUGIN_DIR . 'includes/class-wpvdb-rest.php'; require_once WPVDB_PLUGIN_DIR . 'includes/class-wpvdb-query.php'; require_once WPVDB_PLUGIN_DIR . 'includes/class-wpvdb-settings.php';