Merge branch 'main' into front/setting_page_theme

Author: nsarrazin

.env

@@ -18,6 +18,7 @@ SERPER_API_KEY=#your serper.dev api key here
 SERPAPI_KEY=#your serpapi key here
 SERPSTACK_API_KEY=#your serpstack api key here
 USE_LOCAL_WEBSEARCH=#set to true to parse google results yourself, overrides other API keys
+SEARXNG_QUERY_URL=# where '<query>' will be replaced with query keywords see https://docs.searxng.org/dev/search_api.html eg https://searxng.yourdomain.com/search?q=<query>&engines=duckduckgo,google&format=json
 
 WEBSEARCH_ALLOWLIST=`[]` # if it's defined, allow websites from only this list.
 WEBSEARCH_BLOCKLIST=`[]` # if it's defined, block websites from this list.
@@ -135,4 +136,6 @@ ENABLE_ASSISTANTS=false #set to true to enable assistants feature
 
 ALTERNATIVE_REDIRECT_URLS=`[]` #valide alternative redirect URL for OAuth
 
-WEBHOOK_URL_REPORT_ASSISTANT=#provide webhook url to get notified when an assistant gets reported
+WEBHOOK_URL_REPORT_ASSISTANT=#provide webhook url to get notified when an assistant gets reported
+
+ALLOWED_USER_EMAILS=`[]` # if it's defined, only these emails will be allowed to use the app

README.md

@@ -166,7 +166,7 @@ PUBLIC_APP_DISCLAIMER=
 
 You can enable the web search through an API by adding `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) or `SERPSTACK_API_KEY` ([serpstack.com](https://serpstack.com/)) to your `.env.local`.
 
-You can also simply enable the local websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local`.
+You can also simply enable the local google websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local` or specify a SearXNG instance by adding the query URL to `SEARXNG_QUERY_URL`.
 
 ### Custom models
 

src/lib/server/database.ts

@@ -67,8 +67,11 @@ client.on("open", () => {
 	sharedConversations.createIndex({ hash: 1 }, { unique: true }).catch(console.error);
 	settings.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(console.error);
 	settings.createIndex({ userId: 1 }, { unique: true, sparse: true }).catch(console.error);
+	settings.createIndex({ assistants: 1 }).catch(console.error);
 	users.createIndex({ hfUserId: 1 }, { unique: true }).catch(console.error);
 	users.createIndex({ sessionId: 1 }, { unique: true, sparse: true }).catch(console.error);
+	// No unicity because due to renames & outdated info from oauth provider, there may be the same username on different users
+	users.createIndex({ username: 1 }).catch(console.error);
 	messageEvents.createIndex({ createdAt: 1 }, { expireAfterSeconds: 60 }).catch(console.error);
 	sessions.createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0 }).catch(console.error);
 	sessions.createIndex({ sessionId: 1 }, { unique: true }).catch(console.error);

src/lib/server/endpoints/openai/openAIChatToTextGenerationStream.ts

@@ -22,7 +22,7 @@ export async function* openAIChatToTextGenerationStream(
 				id: tokenId++,
 				text: content ?? "",
 				logprob: 0,
-				special: false,
+				special: last,
 			},
 			generated_text: last ? generatedText : null,
 			details: null,

src/lib/server/endpoints/openai/openAICompletionToTextGenerationStream.ts

@@ -22,7 +22,7 @@ export async function* openAICompletionToTextGenerationStream(
 				id: tokenId++,
 				text,
 				logprob: 0,
-				special: false,
+				special: last,
 			},
 			generated_text: last ? generatedText : null,
 			details: null,

src/lib/server/models.ts

@@ -22,7 +22,7 @@ const modelConfig = z.object({
 	/** Used as an identifier in DB */
 	id: z.string().optional(),
 	/** Used to link to the model page, and for inference */
-	name: z.string().min(1),
+	name: z.string().default(""),
 	displayName: z.string().min(1).optional(),
 	description: z.string().min(1).optional(),
 	websiteUrl: z.string().url().optional(),

src/lib/server/websearch/searchSearxng.ts

@@ -0,0 +1,34 @@
+import { SEARXNG_QUERY_URL } from "$env/static/private";
+
+export async function searchSearxng(query: string) {
+	const abortController = new AbortController();
+	setTimeout(() => abortController.abort(), 10000);
+
+	// Insert the query into the URL template
+	let url = SEARXNG_QUERY_URL.replace("<query>", query);
+
+	// Check if "&format=json" already exists in the URL
+	if (!url.includes("&format=json")) {
+		url += "&format=json";
+	}
+
+	// Call the URL to return JSON data
+	const jsonResponse = await fetch(url, {
+		signal: abortController.signal,
+	})
+		.then((response) => response.json() as Promise<{ results: { url: string }[] }>)
+		.catch((error) => {
+			console.error("Failed to fetch or parse JSON", error);
+			throw new Error("Failed to fetch or parse JSON");
+		});
+
+	// Extract 'url' elements from the JSON response and trim to the top 5 URLs
+	const urls = jsonResponse.results.slice(0, 5).map((item) => item.url);
+
+	if (!urls.length) {
+		throw new Error(`Response doesn't contain any "url" elements`);
+	}
+
+	// Map URLs to the correct object shape
+	return { organic_results: urls.map((link) => ({ link })) };
+}

src/lib/server/websearch/searchWeb.ts

@@ -5,22 +5,33 @@ import {
 	SERPER_API_KEY,
 	SERPSTACK_API_KEY,
 	USE_LOCAL_WEBSEARCH,
+	SEARXNG_QUERY_URL,
 	YDC_API_KEY,
 } from "$env/static/private";
 import { getJson } from "serpapi";
 import type { GoogleParameters } from "serpapi";
 import { searchWebLocal } from "./searchWebLocal";
+import { searchSearxng } from "./searchSearxng";
 
 // get which SERP api is providing web results
 export function getWebSearchProvider() {
-	return YDC_API_KEY ? WebSearchProvider.YOU : WebSearchProvider.GOOGLE;
+	if (YDC_API_KEY) {
+		return WebSearchProvider.YOU;
+	} else if (SEARXNG_QUERY_URL) {
+		return WebSearchProvider.SEARXNG;
+	} else {
+		return WebSearchProvider.GOOGLE;
+	}
 }
 
 // Show result as JSON
 export async function searchWeb(query: string) {
 	if (USE_LOCAL_WEBSEARCH) {
 		return await searchWebLocal(query);
 	}
+	if (SEARXNG_QUERY_URL) {
+		return await searchSearxng(query);
+	}
 	if (SERPER_API_KEY) {
 		return await searchWebSerper(query);
 	}

src/lib/types/WebSearch.ts

@@ -42,4 +42,5 @@ interface YouSearchHit {
 export enum WebSearchProvider {
 	GOOGLE = "Google",
 	YOU = "You.com",
+	SEARXNG = "SearXNG",
 }

src/lib/utils/formatUserCount.ts

@@ -0,0 +1,18 @@
+export function formatUserCount(userCount: number): string {
+	const userCountRanges: { min: number; max: number; label: string }[] = [
+		{ min: 0, max: 1, label: "1" },
+		{ min: 2, max: 9, label: "1-10" },
+		{ min: 10, max: 49, label: "10+" },
+		{ min: 50, max: 99, label: "50+" },
+		{ min: 100, max: 299, label: "100+" },
+		{ min: 300, max: 499, label: "300+" },
+		{ min: 500, max: 999, label: "500+" },
+		{ min: 1_000, max: 2_999, label: "1k+" },
+		{ min: 3_000, max: 4_999, label: "3k+" },
+		{ min: 5_000, max: 9_999, label: "5k+" },
+		{ min: 10_000, max: Infinity, label: "10k+" },
+	];
+
+	const range = userCountRanges.find(({ min, max }) => userCount >= min && userCount <= max);
+	return range?.label ?? "";
+}

src/routes/+layout.server.ts

@@ -12,6 +12,7 @@ import {
 	MESSAGES_BEFORE_LOGIN,
 	YDC_API_KEY,
 	USE_LOCAL_WEBSEARCH,
+	SEARXNG_QUERY_URL,
 	ENABLE_ASSISTANTS,
 } from "$env/static/private";
 import { ObjectId } from "mongodb";
@@ -126,7 +127,8 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
 				SERPER_API_KEY ||
 				SERPSTACK_API_KEY ||
 				YDC_API_KEY ||
-				USE_LOCAL_WEBSEARCH
+				USE_LOCAL_WEBSEARCH ||
+				SEARXNG_QUERY_URL
 			),
 			ethicsModalAccepted: !!settings?.ethicsModalAcceptedAt,
 			ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,

src/routes/+layout.svelte

@@ -157,7 +157,7 @@
 		href="{PUBLIC_ORIGIN || $page.url.origin}{base}/{PUBLIC_APP_ASSETS}/manifest.json"
 	/>
 
-	{#if PUBLIC_PLAUSIBLE_SCRIPT_URL}
+	{#if PUBLIC_PLAUSIBLE_SCRIPT_URL && PUBLIC_ORIGIN}
 		<script
 			defer
 			data-domain={new URL(PUBLIC_ORIGIN).hostname}
@@ -166,7 +166,7 @@
 	{/if}
 </svelte:head>
 
-{#if !$settings.ethicsModalAccepted}
+{#if !$settings.ethicsModalAccepted && $page.url.pathname !== "/privacy"}
 	<DisclaimerModal />
 {/if}
 

src/routes/assistant/[assistantId]/+page.svelte

@@ -96,7 +96,7 @@
 					// `result` is an `ActionResult` object
 					if (result.type === "success") {
 						$settings.activeModel = data.assistant._id;
-						goto(`${base}`);
+						goto(`${base}` || "/");
 					} else {
 						await applyAction(result);
 					}

src/routes/assistants/+page.server.ts

@@ -26,7 +26,7 @@ export const load = async ({ url, locals }) => {
 
 	// fetch the top assistants sorted by user count from biggest to smallest, filter out all assistants with only 1 users. filter by model too if modelId is provided
 	const filter: Filter<Assistant> = {
-		modelId: modelId ?? { $exists: true },
+		...(modelId && { modelId }),
 		...(!createdByCurrentUser && { userCount: { $gt: 1 } }),
 		...(createdByName ? { createdByName } : { featured: true }),
 	};

src/routes/assistants/+page.svelte

@@ -13,8 +13,9 @@
 	import CarbonClose from "~icons/carbon/close";
 	import CarbonArrowUpRight from "~icons/carbon/arrow-up-right";
 	import CarbonEarthAmerica from "~icons/carbon/earth-americas-filled";
-	// import CarbonViewOff from "~icons/carbon/view-off-filled";
+	import CarbonUserMultiple from "~icons/carbon/user-multiple";
 	import Pagination from "$lib/components/Pagination.svelte";
+	import { formatUserCount } from "$lib/utils/formatUserCount";
 	import { getHref } from "$lib/utils/getHref";
 
 	export let data: PageData;
@@ -144,13 +145,16 @@
 			{#each data.assistants as assistant (assistant._id)}
 				<a
 					href="{base}/assistant/{assistant._id}"
-					class="relative flex flex-col items-center justify-center overflow-hidden text-balance rounded-xl border bg-gray-50/50 px-4 py-6 text-center shadow hover:bg-gray-50 hover:shadow-inner max-sm:px-4 sm:h-64 sm:pb-4 dark:border-gray-800/70 dark:bg-gray-950/20 dark:hover:bg-gray-950/40"
+					class="relative flex flex-col items-center justify-center overflow-hidden text-balance rounded-xl border bg-gray-50/50 px-4 py-6 text-center shadow hover:bg-gray-50 hover:shadow-inner max-sm:px-4 sm:h-64 sm:pb-4 xl:pt-8 dark:border-gray-800/70 dark:bg-gray-950/20 dark:hover:bg-gray-950/40"
 				>
-					<!-- {#if assistant.userCount && assistant.userCount > 1}
-						<div class="absolute right-2 top-2" title="share with others to make it public">
-							<CarbonViewOff class="opacity-70" />
+					{#if assistant.userCount && assistant.userCount > 1}
+						<div
+							class="absolute right-3 top-3 flex items-center gap-1 text-xs text-gray-400"
+							title="Number of users"
+						>
+							<CarbonUserMultiple class="text-xxs" />{formatUserCount(assistant.userCount)}
 						</div>
-					{/if} -->
+					{/if}
 					{#if assistant.avatar}
 						<img
 							src="{base}/settings/assistants/{assistant._id}/avatar.jpg"

src/routes/login/callback/+page.server.ts

@@ -3,6 +3,14 @@ import { getOIDCUserData, validateAndParseCsrfToken } from "$lib/server/auth";
 import { z } from "zod";
 import { base } from "$app/paths";
 import { updateUser } from "./updateUser";
+import { ALLOWED_USER_EMAILS } from "$env/static/private";
+import JSON5 from "json5";
+
+const allowedUserEmails = z
+	.array(z.string().email())
+	.optional()
+	.default([])
+	.parse(JSON5.parse(ALLOWED_USER_EMAILS));
 
 export async function load({ url, locals, cookies, request, getClientAddress }) {
 	const { error: errorName, error_description: errorDescription } = z
@@ -33,6 +41,20 @@ export async function load({ url, locals, cookies, request, getClientAddress })
 
 	const { userData } = await getOIDCUserData({ redirectURI: validatedToken.redirectUrl }, code);
 
+	// Filter by allowed user emails
+	if (allowedUserEmails.length > 0) {
+		if (!userData.email) {
+			throw error(403, "User not allowed: email not returned");
+		}
+		const emailVerified = userData.email_verified ?? true;
+		if (!emailVerified) {
+			throw error(403, "User not allowed: email not verified");
+		}
+		if (!allowedUserEmails.includes(userData.email)) {
+			throw error(403, "User not allowed");
+		}
+	}
+
 	await updateUser({
 		userData,
 		locals,

src/routes/settings/assistants/[assistantId]/+page.server.ts

@@ -33,7 +33,9 @@ export const actions: Actions = {
 
 		// and remove it from all users settings
 		await collections.settings.updateMany(
-			{},
+			{
+				assistants: { $in: [assistant._id] },
+			},
 			{
 				$pull: { assistants: assistant._id },
 			}