michal/tit
Browse tree · Show commit · Download archive
Diff
f7abcb5cb650 → a46efea8794c
README.md
Mode 100644 → 100644; object a26271978372 → 20083a337182
@@ -348,3 +348,17 @@ rotation, revocation, stable event selection, and Atom and RSS parsing. Read the [scoped feeds architectural decision record](docs/adr/0019-scoped-feeds.md) for the token, authorization, and ordering contracts. + +## Milestone 4.5 gate + +Run the metadata search gate: + +```text +./scripts/check-m4-5 +``` + +This command tests bounded repository and issue metadata search, public and +private visibility, current collaborator permission, stable result identity, +query validation, and the representative index workload. Read the +[bounded metadata search architectural decision record](docs/adr/0020-bounded-metadata-search.md) +for the limits, authorization, and index decision.
docs/adr/0020-bounded-metadata-search.md
Mode → 100644; object → 03093d87506f
@@ -1,0 +1,64 @@ +# Architectural decision record 0020: Bounded metadata search + +Status: Accepted + +Date: 2026-07-23 + +## Context + +An account must be able to search repository and issue metadata that it can +read. An anonymous user must be able to search public metadata. A permanent +index adds derived state, schema work, and update work. The first search +implementation must show that this additional state is necessary. + +## Decision + +Add a fixed-text, case-insensitive metadata search at `/search`. Search active +repositories, issue titles and bodies, and issue comment bodies. Return one +result for an issue when its issue record or one of its comments matches. Use a +stable repository or issue URL for each result. + +Apply the repository authorization query before the metadata scan. An +anonymous search can read only public repositories. An authenticated search +can also read private repositories that the account owns or has permission to +read. Do not show an archived repository. Check current account and +collaborator state for each search. + +Use these limits for one search: + +- Accept a maximum of 256 query bytes. +- Scan a maximum of 10,000 metadata records. +- Read a maximum of 8 MiB of title and body content. +- Return a maximum of 100 results. +- Stop the application scan after 500 ms. + +Return the completed prefix and tell the user when a limit stops the search. +Run the synchronous SQLite work as a bounded blocking job. Use a GET form that +operates without JavaScript. Read one candidate at a time. Limit each title and +body field to 8 MiB plus one byte in the SQL result. The additional byte shows +that the content limit was reached. Thus, candidate text uses a maximum of 16 +MiB plus two bytes before the application stops the scan. + +Do not add an index. The release-mode workload has one public repository and +9,999 issues. It searches 10,000 records and 449,972 bytes for text in the last +issue. On the development host, the search took 14.4 ms on 2026-07-23. The +index threshold is 250 ms for this workload. Keep the workload in +`tests/metadata_search.rs` so that a contributor can measure it again. +The retained candidate text threshold is 32 MiB. The field limits keep the +implementation below this threshold. + +## Evidence + +The storage test proves that the authorization query filters private metadata. +The search test covers public and private metadata, collaborator permission, +case-insensitive matching, comment result deduplication, query validation, and +stable result identity after a restart. The public-route test covers the form, +anonymous search, authenticated private search, hidden private results, and an +invalid query. The release-mode workload records the index decision. + +## Consequences + +Each request reads canonical SQLite metadata, so search results cannot become +out of date. A large instance can return an incomplete result when it reaches a +limit. Add a derivable embedded index only when repeat measurements exceed the +250 ms time threshold or the 32 MiB retained candidate text threshold.
scripts/check-m4-5
Mode → 100755; object → 1bde6dc58aad
@@ -1,0 +1,7 @@ +#!/bin/sh +set -eu + +./scripts/check +cargo test --locked --release --test metadata_search +cargo test --locked --release --test metadata_search measures_bounded_metadata_search_without_an_index -- --ignored --nocapture +cargo test --locked --release --test public_routes runs_the_complete_issue_workflow_without_javascript
src/http/metadata_search.rs
Mode → 100644; object → 223673f2bc36
@@ -1,0 +1,140 @@
+use askama::Template;
+use axum::Router;
+use axum::extract::{Extension, Query, State};
+use axum::http::StatusCode;
+use axum::response::Response;
+use axum::routing::get;
+use serde::Deserialize;
+
+use crate::search::{MetadataSearchError, MetadataSearchResult};
+use crate::store::StoreError;
+
+use super::{RequestActor, RequestId, WebState, render, render_error};
+
+pub(super) fn routes() -> Router<WebState> {
+ Router::new().route("/search", get(search))
+}
+
+async fn search(
+ State(state): State<WebState>,
+ Extension(request_id): Extension<RequestId>,
+ Extension(actor): Extension<RequestActor>,
+ Query(query): Query<SearchQuery>,
+) -> Response {
+ let Some(value) = query.q else {
+ return search_page(&request_id.0, "", None);
+ };
+ let Some(service) = state.search.clone() else {
+ return search_internal(&request_id.0);
+ };
+ let value_for_search = value.clone();
+ let result = search_job(state, move || {
+ service.search(actor.0.as_deref(), &value_for_search)
+ })
+ .await;
+ match result {
+ Ok(outcome) => search_page(&request_id.0, &value, Some(outcome)),
+ Err(MetadataSearchError::InvalidQuery | MetadataSearchError::Auth(_)) => render_error(
+ StatusCode::BAD_REQUEST,
+ &request_id.0,
+ "Search error",
+ "The metadata search query is not valid.",
+ ),
+ Err(_) => search_internal(&request_id.0),
+ }
+}
+
+async fn search_job<T: Send + 'static>(
+ state: WebState,
+ operation: impl FnOnce() -> Result<T, MetadataSearchError> + Send + 'static,
+) -> Result<T, MetadataSearchError> {
+ let permit = state.jobs.acquire_owned().await.map_err(|_| {
+ MetadataSearchError::Store(StoreError::Integrity(
+ "search worker pool is unavailable".to_owned(),
+ ))
+ })?;
+ tokio::task::spawn_blocking(move || {
+ let _permit = permit;
+ operation()
+ })
+ .await
+ .map_err(|_| {
+ MetadataSearchError::Store(StoreError::Integrity("search worker failed".to_owned()))
+ })?
+}
+
+fn search_page(
+ request_id: &str,
+ query: &str,
+ outcome: Option<crate::search::MetadataSearchOutcome>,
+) -> Response {
+ let searched = outcome.is_some();
+ let (rows_scanned, bytes_scanned, truncated, results) = outcome.map_or_else(
+ || (0, 0, false, Vec::new()),
+ |outcome| {
+ debug_assert_eq!(outcome.query, query.trim());
+ (
+ outcome.rows_scanned,
+ outcome.bytes_scanned,
+ outcome.truncated,
+ outcome.results,
+ )
+ },
+ );
+ render(
+ StatusCode::OK,
+ &MetadataSearchTemplate {
+ request_id,
+ query,
+ searched,
+ rows_scanned,
+ bytes_scanned,
+ truncated,
+ results: results.iter().map(result_view).collect(),
+ },
+ )
+}
+
+fn result_view(result: &MetadataSearchResult) -> MetadataSearchResultView<'_> {
+ MetadataSearchResultView {
+ kind: result.kind,
+ url: &result.url,
+ title: &result.title,
+ summary: &result.summary,
+ stable_id: &result.stable_id,
+ }
+}
+
+fn search_internal(request_id: &str) -> Response {
+ render_error(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ request_id,
+ "Search error",
+ "The metadata search could not be completed.",
+ )
+}
+
+#[derive(Deserialize)]
+struct SearchQuery {
+ q: Option<String>,
+}
+
+struct MetadataSearchResultView<'a> {
+ kind: &'static str,
+ url: &'a str,
+ title: &'a str,
+ summary: &'a str,
+ stable_id: &'a str,
+}
+
+#[derive(Template)]
+#[template(path = "metadata-search.html")]
+struct MetadataSearchTemplate<'a> {
+ request_id: &'a str,
+ query: &'a str,
+ searched: bool,
+ rows_scanned: usize,
+ bytes_scanned: usize,
+ truncated: bool,
+ results: Vec<MetadataSearchResultView<'a>>,
+}
src/http/mod.rs
Mode 100644 → 100644; object e4e965678e3a → 75668e8aa829
@@ -1,5 +1,6 @@
mod feeds;
mod issues;
+mod metadata_search;
mod public;
mod watches;
@@ -26,6 +27,7 @@
use crate::feed_token::FeedTokenService;
use crate::issue::IssueService;
use crate::repository::{RepositoryService, RepositoryServiceError};
+use crate::search::MetadataSearchService;
use crate::session::{SessionError, WebLoginService};
use crate::store::StoreError;
use crate::watch::WatchService;
@@ -50,6 +52,7 @@
repositories: Option<RepositoryService>,
issues: Option<IssueService>,
feeds: Option<FeedTokenService>,
+ search: Option<MetadataSearchService>,
watches: Option<WatchService>,
secure_cookies: bool,
}
@@ -85,6 +88,7 @@
repositories: None,
issues: None,
feeds: None,
+ search: None,
watches: None,
secure_cookies: false,
},
@@ -123,6 +127,7 @@
let repositories = RepositoryService::new(public.database(), public.repository_root());
let issues = IssueService::new(public.database());
let feeds = FeedTokenService::new(public.database());
+ let search = MetadataSearchService::new(public.database());
let watches = WatchService::new(public.database());
Self::start_with_state(
address,
@@ -135,6 +140,7 @@
repositories: Some(repositories),
issues: Some(issues),
feeds: Some(feeds),
+ search: Some(search),
watches: Some(watches),
secure_cookies,
},
@@ -181,13 +187,15 @@
repositories: None,
issues: None,
feeds: None,
+ search: None,
watches: None,
secure_cookies: false,
})
}
fn router_with_state(state: WebState) -> Router {
- let repository_routes = watches::routes()
+ let repository_routes = metadata_search::routes()
+ .merge(watches::routes())
.merge(issues::routes())
.merge(public::routes())
.layer(middleware::from_fn_with_state(
src/main.rs
Mode 100644 → 100644; object ba1d87e7422f → a17845a01076
@@ -21,6 +21,7 @@ mod markdown; mod policy; mod repository; +mod search; mod serve; mod session; #[allow(dead_code, reason = "the server uses only part of the shared SSH API")]
src/search.rs
Mode → 100644; object → bafe8d66532e
@@ -1,0 +1,180 @@
+use std::collections::BTreeSet;
+use std::path::{Path, PathBuf};
+use std::time::{Duration, Instant};
+
+use thiserror::Error;
+
+use crate::auth::{AuthError, validate_username};
+use crate::store::{MetadataSearchCandidate, Store, StoreError};
+
+pub(crate) const MAX_QUERY_BYTES: usize = 256;
+pub(crate) const MAX_SCAN_ROWS: usize = 10_000;
+pub(crate) const MAX_SCAN_BYTES: usize = 8 * 1024 * 1024;
+pub(crate) const MAX_RESULTS: usize = 100;
+pub(crate) const MAX_DURATION: Duration = Duration::from_millis(500);
+
+#[derive(Clone)]
+pub(crate) struct MetadataSearchService {
+ database: PathBuf,
+}
+
+impl MetadataSearchService {
+ pub(crate) fn new(database: &Path) -> Self {
+ Self {
+ database: database.to_owned(),
+ }
+ }
+
+ pub(crate) fn search(
+ &self,
+ actor: Option<&str>,
+ query: &str,
+ ) -> Result<MetadataSearchOutcome, MetadataSearchError> {
+ if let Some(actor) = actor {
+ validate_username(actor)?;
+ }
+ let query = query.trim();
+ if query.is_empty() || query.len() > MAX_QUERY_BYTES || query.chars().any(char::is_control)
+ {
+ return Err(MetadataSearchError::InvalidQuery);
+ }
+ let started = Instant::now();
+ let store = Store::open(&self.database)?;
+ let needle = query.to_lowercase();
+ let mut bytes_scanned = 0_usize;
+ let mut rows_scanned = 0_usize;
+ let mut seen = BTreeSet::new();
+ let mut results = Vec::new();
+ let mut candidate_error = None;
+ let truncated = store.visit_metadata_search_candidates(
+ actor,
+ MAX_SCAN_ROWS,
+ MAX_SCAN_BYTES,
+ |candidate| {
+ if started.elapsed() >= MAX_DURATION {
+ return false;
+ }
+ let Some(bytes) = candidate.title.len().checked_add(candidate.body.len()) else {
+ candidate_error = Some(MetadataSearchError::Limit);
+ return false;
+ };
+ if bytes_scanned.saturating_add(bytes) > MAX_SCAN_BYTES {
+ return false;
+ }
+ bytes_scanned += bytes;
+ rows_scanned += 1;
+ let (Ok(title), Ok(body)) = (
+ std::str::from_utf8(&candidate.title),
+ std::str::from_utf8(&candidate.body),
+ ) else {
+ candidate_error = Some(MetadataSearchError::StoredCandidate);
+ return false;
+ };
+ if !title.to_lowercase().contains(&needle) && !body.to_lowercase().contains(&needle)
+ {
+ return true;
+ }
+ let Ok(result) = result_from_candidate(candidate) else {
+ candidate_error = Some(MetadataSearchError::StoredCandidate);
+ return false;
+ };
+ if !seen.insert(result.url.clone()) {
+ return true;
+ }
+ if results.len() == MAX_RESULTS {
+ return false;
+ }
+ results.push(result);
+ true
+ },
+ )?;
+ if let Some(error) = candidate_error {
+ return Err(error);
+ }
+
+ Ok(MetadataSearchOutcome {
+ query: query.to_owned(),
+ rows_scanned,
+ bytes_scanned,
+ truncated,
+ results,
+ })
+ }
+}
+
+fn result_from_candidate(
+ candidate: MetadataSearchCandidate,
+) -> Result<MetadataSearchResult, MetadataSearchError> {
+ let title =
+ String::from_utf8(candidate.title).map_err(|_| MetadataSearchError::StoredCandidate)?;
+ let body =
+ String::from_utf8(candidate.body).map_err(|_| MetadataSearchError::StoredCandidate)?;
+ let (kind, url, title) = match candidate.kind.as_str() {
+ "repository" => (
+ "Repository",
+ format!("/{}/{}", candidate.owner, candidate.repository),
+ title,
+ ),
+ "issue" | "issue-comment" => {
+ let number = candidate
+ .issue_number
+ .ok_or(MetadataSearchError::StoredCandidate)?;
+ (
+ "Issue",
+ format!(
+ "/{}/{}/issues/{number}",
+ candidate.owner, candidate.repository
+ ),
+ format!(
+ "{}/{} #{number}: {}",
+ candidate.owner, candidate.repository, title
+ ),
+ )
+ }
+ _ => return Err(MetadataSearchError::StoredCandidate),
+ };
+ let summary = body
+ .split_whitespace()
+ .collect::<Vec<_>>()
+ .join(" ")
+ .chars()
+ .take(160)
+ .collect();
+ Ok(MetadataSearchResult {
+ kind,
+ url,
+ title,
+ summary,
+ stable_id: candidate.record_id,
+ })
+}
+
+pub(crate) struct MetadataSearchOutcome {
+ pub(crate) query: String,
+ pub(crate) rows_scanned: usize,
+ pub(crate) bytes_scanned: usize,
+ pub(crate) truncated: bool,
+ pub(crate) results: Vec<MetadataSearchResult>,
+}
+
+pub(crate) struct MetadataSearchResult {
+ pub(crate) kind: &'static str,
+ pub(crate) url: String,
+ pub(crate) title: String,
+ pub(crate) summary: String,
+ pub(crate) stable_id: String,
+}
+
+#[derive(Debug, Error)]
+pub(crate) enum MetadataSearchError {
+ #[error(transparent)]
+ Auth(#[from] AuthError),
+ #[error(transparent)]
+ Store(#[from] StoreError),
+ #[error("metadata search query is not valid")]
+ InvalidQuery,
+ #[error("metadata search reached an arithmetic limit")]
+ Limit,
+ #[error("stored metadata search candidate is not valid")]
+ StoredCandidate,
+}
src/store/mod.rs
Mode 100644 → 100644; object f95154811278 → 6b9c5adaa79c
@@ -2140,6 +2140,86 @@
})
}
+ pub(crate) fn visit_metadata_search_candidates(
+ &self,
+ actor: Option<&str>,
+ limit: usize,
+ max_field_bytes: usize,
+ mut visit: impl FnMut(MetadataSearchCandidate) -> bool,
+ ) -> Result<bool, StoreError> {
+ let query_limit = limit.checked_add(1).ok_or(StoreError::EventLimit)?;
+ let query_limit = i64::try_from(query_limit).map_err(|_| StoreError::EventLimit)?;
+ let max_field_bytes = max_field_bytes
+ .checked_add(1)
+ .and_then(|value| i64::try_from(value).ok())
+ .ok_or(StoreError::EventLimit)?;
+ let mut statement = self.connection.prepare(
+ "WITH visible_repository AS (
+ SELECT repository.id, owner.username AS owner, repository.slug
+ FROM repository
+ JOIN account AS owner ON owner.id = repository.owner_account_id
+ LEFT JOIN account AS actor
+ ON actor.username = ?1 AND actor.state = 'active'
+ WHERE repository.state = 'active'
+ AND (repository.visibility = 'public'
+ OR repository.owner_account_id = actor.id
+ OR EXISTS (
+ SELECT 1 FROM repository_collaborator
+ WHERE repository_collaborator.repository_id = repository.id
+ AND repository_collaborator.account_id = actor.id
+ ))
+ ), candidates AS (
+ SELECT 'repository' AS kind, visible_repository.id AS record_id,
+ visible_repository.owner, visible_repository.slug,
+ NULL AS issue_number,
+ CAST(visible_repository.owner || '/' || visible_repository.slug AS BLOB)
+ AS title,
+ X'' AS body
+ FROM visible_repository
+ UNION ALL
+ SELECT 'issue', issue.id, visible_repository.owner,
+ visible_repository.slug, issue.number,
+ COALESCE(substr(CAST(issue.title AS BLOB), 1, ?3), X''),
+ COALESCE(substr(CAST(issue.body AS BLOB), 1, ?3), X'')
+ FROM issue
+ JOIN visible_repository ON visible_repository.id = issue.repository_id
+ UNION ALL
+ SELECT 'issue-comment', issue_comment.id, visible_repository.owner,
+ visible_repository.slug, issue.number,
+ COALESCE(substr(CAST(issue.title AS BLOB), 1, ?3), X''),
+ COALESCE(substr(CAST(issue_comment.body AS BLOB), 1, ?3), X'')
+ FROM issue_comment
+ JOIN issue ON issue.id = issue_comment.issue_id
+ JOIN visible_repository ON visible_repository.id = issue.repository_id
+ )
+ SELECT kind, record_id, owner, slug, issue_number, title, body
+ FROM candidates
+ ORDER BY owner, slug, kind, issue_number, record_id
+ LIMIT ?2",
+ )?;
+ let mut rows = statement.query(rusqlite::params![actor, query_limit, max_field_bytes])?;
+ let mut visited = 0_usize;
+ while let Some(row) = rows.next()? {
+ if visited == limit {
+ return Ok(true);
+ }
+ visited += 1;
+ let candidate = MetadataSearchCandidate {
+ kind: row.get(0)?,
+ record_id: row.get(1)?,
+ owner: row.get(2)?,
+ repository: row.get(3)?,
+ issue_number: row.get(4)?,
+ title: row.get(5)?,
+ body: row.get(6)?,
+ };
+ if !visit(candidate) {
+ return Ok(true);
+ }
+ }
+ Ok(false)
+ }
+
#[allow(
dead_code,
reason = "some integration tests compile storage without authorization"
@@ -2610,6 +2690,16 @@
pub(crate) username: String,
pub(crate) target: Option<String>,
pub(crate) events: Vec<ActivityEventRecord>,
+}
+
+pub(crate) struct MetadataSearchCandidate {
+ pub(crate) kind: String,
+ pub(crate) record_id: String,
+ pub(crate) owner: String,
+ pub(crate) repository: String,
+ pub(crate) issue_number: Option<i64>,
+ pub(crate) title: Vec<u8>,
+ pub(crate) body: Vec<u8>,
}
pub(crate) struct GitOperationIntent<'a> {
templates/base.html
Mode 100644 → 100644; object 3242bac8ffe1 → 250279dbab58
@@ -12,6 +12,7 @@
<a class="brand" href="/" aria-label="tit home">tit</a>
<nav aria-label="Primary">
<a href="/">Home</a>
+ <a href="/search">Search</a>
<a href="/signup">Create account</a>
<a href="/recover">Recover account</a>
<a href="/login">Log in</a>
templates/metadata-search.html
Mode → 100644; object → 4113edd31490
@@ -1,0 +1,31 @@
+{% extends "base.html" %}
+{% block title %}Search · tit{% endblock %}
+{% block content %}
+ <h1>Search repositories and issues</h1>
+ <form method="get" action="/search">
+ <label for="metadata-query">Search text</label>
+ <input id="metadata-query" name="q" value="{{ query }}" maxlength="256" required>
+ <button type="submit">Search</button>
+ </form>
+{% if searched %}
+ <p>Scanned {{ rows_scanned }} records and {{ bytes_scanned }} bytes.</p>
+{% if truncated %}
+ <p>The result is incomplete because the search reached a limit.</p>
+{% endif %}
+{% if results.is_empty() %}
+ <p>No repository or issue matched the search text.</p>
+{% else %}
+ <ol class="search-results">
+{% for result in results %}
+ <li data-result-id="{{ result.stable_id }}">
+ <span>{{ result.kind }}</span>
+ <a href="{{ result.url }}">{{ result.title }}</a>
+{% if !result.summary.is_empty() %}
+ <p>{{ result.summary }}</p>
+{% endif %}
+ </li>
+{% endfor %}
+ </ol>
+{% endif %}
+{% endif %}
+{% endblock %}
tests/metadata_search.rs
Mode → 100644; object → 6d18d024bc6a
@@ -1,0 +1,245 @@
+#[allow(dead_code, reason = "the search test uses only username validation")]
+#[path = "../src/auth.rs"]
+mod auth;
+#[path = "../src/domain/mod.rs"]
+mod domain;
+#[path = "../src/search.rs"]
+mod search;
+#[allow(dead_code, reason = "the search test uses only metadata storage")]
+#[path = "../src/store/mod.rs"]
+mod store;
+
+use std::time::{Duration, Instant};
+
+use search::{MetadataSearchError, MetadataSearchService};
+use store::{InitialAdministrator, NewIssue, NewRepository, RepositoryOrigin, Store};
+use tempfile::TempDir;
+
+#[test]
+fn searches_only_authorized_repository_and_issue_metadata_with_limits() {
+ let directory = TempDir::new().expect("create a search fixture directory");
+ let database = directory.path().join("tit.sqlite3");
+ let mut store = Store::open(&database).expect("create the search database");
+ store
+ .create_initial_administrator(&InitialAdministrator {
+ username: "alice",
+ canonical_key: "ssh-ed25519 AAAAalice",
+ fingerprint: "SHA256:alice",
+ recovery_hash: &[1; 32],
+ created_at: 1,
+ })
+ .expect("create the search owner");
+ store
+ .connection()
+ .execute_batch(
+ "INSERT INTO account (id, username, is_administrator, state, created_at)
+ VALUES (2, 'bob', 0, 'active', 1),
+ (3, 'stranger', 0, 'active', 1);",
+ )
+ .expect("create search accounts");
+ create_repository(
+ &mut store,
+ "11111111111111111111111111111111",
+ "public-project",
+ );
+ create_repository(
+ &mut store,
+ "22222222222222222222222222222222",
+ "private-project",
+ );
+ store
+ .connection()
+ .execute_batch(
+ "UPDATE repository SET visibility = 'private'
+ WHERE slug = 'private-project';
+ INSERT INTO repository_collaborator
+ (repository_id, account_id, role, created_at)
+ VALUES ('22222222222222222222222222222222', 2, 'reader', 2);",
+ )
+ .expect("make one repository private");
+ store
+ .create_issue(&NewIssue {
+ owner: "alice",
+ repository: "public-project",
+ actor: "alice",
+ title: "Public Needle",
+ body: "The public body contains metadata.",
+ created_at: 3,
+ })
+ .expect("create a public issue");
+ store
+ .comment_issue(
+ "alice",
+ "public-project",
+ 1,
+ "alice",
+ "A needle in a comment must not duplicate the issue result.",
+ 4,
+ )
+ .expect("create a matching comment");
+ store
+ .create_issue(&NewIssue {
+ owner: "alice",
+ repository: "private-project",
+ actor: "alice",
+ title: "Private Secret",
+ body: "Only a repository reader can find this.",
+ created_at: 5,
+ })
+ .expect("create a private issue");
+ store
+ .connection()
+ .execute_batch(
+ "WITH RECURSIVE sequence(number) AS (
+ VALUES (2)
+ UNION ALL
+ SELECT number + 1 FROM sequence WHERE number < 102
+ )
+ INSERT INTO issue
+ (id, repository_id, number, title, body, state,
+ author_account_id, created_at, updated_at, closed_at)
+ SELECT printf('%032x', number + 1000),
+ '11111111111111111111111111111111',
+ number, printf('Bulk match %03d', number), '',
+ 'open', 1, number + 10, number + 10, NULL
+ FROM sequence;",
+ )
+ .expect("create enough matches to reach the result limit");
+ drop(store);
+
+ let service = MetadataSearchService::new(&database);
+ let public = service
+ .search(None, "NEEDLE")
+ .expect("search public metadata");
+ assert_eq!(public.results.len(), 1);
+ assert_eq!(public.query, "NEEDLE");
+ assert_eq!(public.results[0].kind, "Issue");
+ assert_eq!(public.results[0].url, "/alice/public-project/issues/1");
+ assert!(public.results[0].title.contains("Public Needle"));
+ assert!(public.results[0].summary.contains("public body"));
+ assert_eq!(public.results[0].stable_id.len(), 32);
+ assert!(!public.truncated);
+ assert!(public.rows_scanned >= 3);
+ assert!(public.bytes_scanned > 0);
+
+ let bounded = service
+ .search(None, "bulk match")
+ .expect("search more results than the output limit");
+ assert_eq!(bounded.results.len(), 100);
+ assert!(bounded.truncated);
+
+ let reader = service
+ .search(Some("bob"), "private secret")
+ .expect("search private metadata as a reader");
+ assert_eq!(reader.results.len(), 1);
+ assert_eq!(reader.results[0].url, "/alice/private-project/issues/1");
+ assert!(
+ service
+ .search(Some("stranger"), "private secret")
+ .expect("search private metadata as a stranger")
+ .results
+ .is_empty()
+ );
+
+ let restarted = MetadataSearchService::new(&database)
+ .search(None, "needle")
+ .expect("repeat metadata search after restart");
+ assert_eq!(
+ restarted
+ .results
+ .iter()
+ .map(|result| (&result.stable_id, &result.url))
+ .collect::<Vec<_>>(),
+ public
+ .results
+ .iter()
+ .map(|result| (&result.stable_id, &result.url))
+ .collect::<Vec<_>>()
+ );
+ for query in ["", "line\nbreak"] {
+ assert!(matches!(
+ service.search(None, query),
+ Err(MetadataSearchError::InvalidQuery)
+ ));
+ }
+ assert!(matches!(
+ service.search(None, &"x".repeat(257)),
+ Err(MetadataSearchError::InvalidQuery)
+ ));
+}
+
+#[test]
+#[ignore = "M4.5 representative metadata search measurement"]
+fn measures_bounded_metadata_search_without_an_index() {
+ let directory = TempDir::new().expect("create a search measurement directory");
+ let database = directory.path().join("tit.sqlite3");
+ let mut store = Store::open(&database).expect("create the search measurement database");
+ store
+ .create_initial_administrator(&InitialAdministrator {
+ username: "alice",
+ canonical_key: "ssh-ed25519 AAAAalice",
+ fingerprint: "SHA256:alice",
+ recovery_hash: &[1; 32],
+ created_at: 1,
+ })
+ .expect("create the search measurement owner");
+ create_repository(
+ &mut store,
+ "33333333333333333333333333333333",
+ "measurement",
+ );
+ store
+ .connection()
+ .execute_batch(
+ "WITH RECURSIVE sequence(number) AS (
+ VALUES (1)
+ UNION ALL
+ SELECT number + 1 FROM sequence WHERE number < 9999
+ )
+ INSERT INTO issue
+ (id, repository_id, number, title, body, state,
+ author_account_id, created_at, updated_at, closed_at)
+ SELECT printf('%032x', number),
+ '33333333333333333333333333333333',
+ number,
+ printf('Issue %05d', number),
+ printf('Representative metadata body %05d', number),
+ 'open', 1, number + 2, number + 2, NULL
+ FROM sequence;",
+ )
+ .expect("create representative metadata");
+ drop(store);
+
+ let started = Instant::now();
+ let outcome = MetadataSearchService::new(&database)
+ .search(None, "representative metadata body 09999")
+ .expect("measure metadata search");
+ let elapsed = started.elapsed();
+ assert_eq!(outcome.results.len(), 1);
+ assert_eq!(outcome.rows_scanned, 10_000);
+ assert!(!outcome.truncated);
+ assert!(
+ elapsed < Duration::from_millis(250),
+ "metadata scan exceeded the 250 ms index threshold: {elapsed:?}"
+ );
+ eprintln!(
+ "searched {} records and {} bytes in {elapsed:?}",
+ outcome.rows_scanned, outcome.bytes_scanned
+ );
+}
+
+fn create_repository(store: &mut Store, id: &str, slug: &str) {
+ store
+ .create_repository(&NewRepository {
+ id,
+ owner: "alice",
+ slug,
+ object_format: "sha1",
+ created_at: 2,
+ origin: RepositoryOrigin::Created,
+ initial_references: &[],
+ actor: "alice",
+ correlation_id: "metadata-search",
+ })
+ .expect("create a search repository");
+}
tests/public_routes.rs
Mode 100644 → 100644; object 8ae26cd0e021 → 820d313f2a61
@@ -48,6 +48,8 @@
)]
#[path = "../src/repository.rs"]
mod repository;
+#[path = "../src/search.rs"]
+mod search;
#[allow(dead_code, reason = "the public-route test does not complete a login")]
#[path = "../src/session.rs"]
mod session;
@@ -727,6 +729,26 @@
assert!(final_text.contains("Assignees: <span>alice</span>"));
assert!(final_text.contains("issue-created"));
assert!(final_text.contains("issue-reopened"));
+ let search_page = request(server.address(), "GET", "/search", &[], &[]);
+ assert_eq!(search_page.status, 200);
+ assert!(
+ search_page
+ .text()
+ .contains("Search repositories and issues")
+ );
+ let metadata_search = request(
+ server.address(),
+ "GET",
+ "/search?q=Edited%20issue",
+ &[],
+ &[],
+ );
+ assert_eq!(metadata_search.status, 200);
+ assert!(metadata_search.text().contains("/alice/example/issues/1"));
+ assert_eq!(
+ request(server.address(), "GET", "/search?q=", &[], &[]).status,
+ 400
+ );
let feed = request(server.address(), "GET", "/alice/example/atom.xml", &[], &[]);
assert_eq!(feed.status, 200);
assert!(feed.text().contains("alice reopened #1"));
@@ -872,6 +894,30 @@
)
.status,
404
+ );
+ let anonymous_private_search = request(
+ server.address(),
+ "GET",
+ "/search?q=Edited%20issue",
+ &[],
+ &[],
+ );
+ assert!(
+ anonymous_private_search
+ .text()
+ .contains("No repository or issue matched")
+ );
+ let owner_private_search = request(
+ server.address(),
+ "GET",
+ "/search?q=Edited%20issue",
+ &[("Cookie", cookie.as_str())],
+ &[],
+ );
+ assert!(
+ owner_private_search
+ .text()
+ .contains("/alice/example/issues/1")
);
assert_eq!(
request(server.address(), "GET", &private_path, &[], &[]).status,
tests/snapshots/web/bad-request.html
Mode 100644 → 100644; object cb36f34dd1de → ff6cd3e01450
@@ -12,6 +12,7 @@
<a class="brand" href="/" aria-label="tit home">tit</a>
<nav aria-label="Primary">
<a href="/">Home</a>
+ <a href="/search">Search</a>
<a href="/signup">Create account</a>
<a href="/recover">Recover account</a>
<a href="/login">Log in</a>
tests/snapshots/web/home.html
Mode 100644 → 100644; object 64f08cd9b85c → 1a9666ebe846
@@ -12,6 +12,7 @@
<a class="brand" href="/" aria-label="tit home">tit</a>
<nav aria-label="Primary">
<a href="/">Home</a>
+ <a href="/search">Search</a>
<a href="/signup">Create account</a>
<a href="/recover">Recover account</a>
<a href="/login">Log in</a>
tests/snapshots/web/method-not-allowed.html
Mode 100644 → 100644; object cad969198904 → b7bab2ebc3c5
@@ -12,6 +12,7 @@
<a class="brand" href="/" aria-label="tit home">tit</a>
<nav aria-label="Primary">
<a href="/">Home</a>
+ <a href="/search">Search</a>
<a href="/signup">Create account</a>
<a href="/recover">Recover account</a>
<a href="/login">Log in</a>
tests/snapshots/web/not-found.html
Mode 100644 → 100644; object 3d27c4f01d27 → 804649ae1202
@@ -12,6 +12,7 @@
<a class="brand" href="/" aria-label="tit home">tit</a>
<nav aria-label="Primary">
<a href="/">Home</a>
+ <a href="/search">Search</a>
<a href="/signup">Create account</a>
<a href="/recover">Recover account</a>
<a href="/login">Log in</a>
tests/sqlite.rs
Mode 100644 → 100644; object 094b2b018aa7 → 99363a0f2034
@@ -1119,6 +1119,36 @@
assert_eq!(watched_token.scope, "watched");
assert_eq!(assignment_token.scope, "assignments");
assert_eq!(mention_token.scope, "mentions");
+ let mut metadata = Vec::new();
+ let metadata_truncated = store
+ .visit_metadata_search_candidates(Some("bob"), 100, 1024, |candidate| {
+ metadata.push(candidate);
+ true
+ })
+ .expect("scan authorized metadata");
+ assert!(!metadata_truncated);
+ assert!(metadata.iter().any(|candidate| {
+ candidate.kind == "repository"
+ && candidate.record_id == "00112233445566778899aabbccddeeff"
+ && candidate.owner == "alice"
+ && candidate.repository == "project"
+ && candidate.title == b"alice/project"
+ && candidate.body.is_empty()
+ }));
+ assert!(
+ metadata
+ .iter()
+ .any(|candidate| candidate.issue_number == Some(1))
+ );
+ let mut anonymous_metadata = Vec::new();
+ let anonymous_truncated = store
+ .visit_metadata_search_candidates(None, 100, 1024, |candidate| {
+ anonymous_metadata.push(candidate);
+ true
+ })
+ .expect("scan anonymous metadata");
+ assert!(!anonymous_truncated);
+ assert!(anonymous_metadata.is_empty());
let comments_before: i64 = store
.connection()
tests/web_shell.rs
Mode 100644 → 100644; object 6340278918c8 → 167a67e6aa7d
@@ -43,6 +43,8 @@ #[allow(dead_code, reason = "the Web shell test does not create repositories")] #[path = "../src/repository.rs"] mod repository; +#[path = "../src/search.rs"] +mod search; #[allow(dead_code, reason = "the Web shell test does not complete a login")] #[path = "../src/session.rs"] mod session;