From 252ea713b1376b2fd82c8fd868344811a629b686 Mon Sep 17 00:00:00 2001 From: Mo Elzubeir Date: Fri, 29 May 2026 13:10:05 -0500 Subject: [PATCH] feat: add entity analytics, pluggable cache, and pitfalls docs - Add Cache interface with MemoryCache and NoopCache implementations - Make SocialhoseClient accept injectable cache option - Remove Next.js next.revalidate coupling from transport - Add entity analytics: getEntityBrief, getEntityStats, getEntityBriefs, getCampaignIdByMatch with exact sentiment/platform faceting and cumulative-differenced timeline - Add 23 new tests (29 total) covering entity analytics, cache injection, sentiment reconciliation, bounded concurrency, and timeline differencing - Update README with entity analytics, custom caching, and pitfalls sections - Fix CI branch: main -> master --- .github/workflows/javascript.yml | 2 +- pnpm-lock.yaml | 2 +- sdks/javascript/README.md | 85 +++++- sdks/javascript/package.json | 26 +- sdks/javascript/src/cache.ts | 41 +++ sdks/javascript/src/index.ts | 297 ++++++++++++++++++-- sdks/javascript/test/client.test.ts | 413 +++++++++++++++++++++++++++- 7 files changed, 836 insertions(+), 30 deletions(-) create mode 100644 sdks/javascript/src/cache.ts diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml index c04b462..ffc5f31 100644 --- a/.github/workflows/javascript.yml +++ b/.github/workflows/javascript.yml @@ -9,7 +9,7 @@ on: - 'pnpm-workspace.yaml' - '.github/workflows/javascript.yml' push: - branches: [main] + branches: [master] paths: - 'sdks/javascript/**' - 'package.json' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6782a20..0b41de1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,7 +17,7 @@ importers: specifier: 8.3.0 version: 8.3.0(postcss@8.5.15)(typescript@5.6.2) typescript: - specifier: 5.6.2 + specifier: ^5.6.2 version: 5.6.2 vitest: specifier: 2.1.1 diff --git a/sdks/javascript/README.md b/sdks/javascript/README.md index bb85815..5efdea5 100644 --- a/sdks/javascript/README.md +++ b/sdks/javascript/README.md @@ -44,8 +44,11 @@ The SDK sends `Authorization: Api-Key ` and a browser-like `User-Agent` by ## Endpoints +### Campaigns - `getCampaigns()` - `getCampaign(id)` + +### Analytics - `getOverview(filters)` - `getTimeline(filters)` - `getSentiment(filters)` @@ -54,11 +57,23 @@ The SDK sends `Authorization: Api-Key ` and a browser-like `User-Agent` by - `getTopKeywords(filters)` - `getTrending(filters)` - `getTopMentions(filters)` + +### Mentions - `getMentions(filters)` + +### Mailing Lists - `getMailingLists()` - `inviteMailingListMember(listId, invite)` -- `get(path, params)` for lower-level GET access -- `post(path, body)` for lower-level POST access + +### Entity Analytics +- `getEntityBrief(term, campaignId?)` — one request: count + top-20 engagement sample with derived sentiment/platform +- `getEntityStats(term, campaignId?)` — full dashboard: exact sentiment faceting, exact platform mix, 14-day cumulative-differenced timeline, 7d momentum +- `getEntityBriefs(terms, campaignId?, concurrency?)` — batch entity resolution with bounded concurrency (default 20) +- `getCampaignIdByMatch(substring)` — resolve a live campaign ID by matching its name + +### Low-Level +- `get(path, params)` for direct GET access +- `post(path, body)` for direct POST access ## Filtering examples @@ -77,14 +92,74 @@ await socialhose.getTimeline({ }); ``` -## Next.js cache integration +## Custom Caching -Pass `revalidateSeconds` per request. In Next.js this is forwarded as `fetch(..., { next: { revalidate } })`; outside Next.js it is harmless. +The SDK ships with `MemoryCache` (in-memory, per-entry TTL) and `NoopCache` (no caching). You can inject your own by implementing the `Cache` interface: ```ts -await socialhose.getMentions({ content_search: 'ozempic' }, { revalidateSeconds: 3600 }); +import { SocialhoseClient, Cache } from '@socialhose/api'; + +class RedisCache implements Cache { + async get(key: string) { /* redis.get(key) */ } + async set(key: string, value: unknown, ttlMs: number) { /* redis.set(key, value, 'PX', ttlMs) */ } + async delete(key: string) { /* redis.del(key) */ } +} + +const socialhose = new SocialhoseClient({ + apiKey: process.env.SOCIALHOSE_API_KEY!, + cache: new RedisCache(), +}); ``` +Pass `revalidateSeconds` per request to control per-call TTL in your cache implementation: + +```ts +await socialhose.getMentions( + { content_search: 'ozempic' }, + { revalidateSeconds: 3600 }, +); +``` + +## Entity Analytics + +Search across all mentions for a specific term, person, or organization: + +```ts +// Quick count + top mentions +const brief = await socialhose.getEntityBrief('Burhan'); +console.log(brief.total, brief.sentiment, brief.platformMix); + +// Full dashboard: exact distributions, timeline, momentum +const stats = await socialhose.getEntityStats('RSF', 'campaign-id'); +console.log( + stats.total, + stats.momentumPct, // last 7 days vs prior 7 + stats.sentiment, // exact (facets reconcile) or estimated (from sample) + stats.sparkline, // 14-day daily volume +); + +// Batch resolve many entities with bounded concurrency +const briefs = await socialhose.getEntityBriefs( + ['Burhan', 'Hemedti', 'SAF', 'RSF'], + 'campaign-id', + 10, // concurrency +); +``` + +Entity analytics fan out multiple requests per entity (sentiment faceting: 3 calls; platform mix: 6 calls; timeline: 15 calls). Set `cacheTtlMs` or inject a persistent cache to stay under the ~60 req/min rate limit. + +## Pitfalls + +**Cloudflare UA blocking.** The Socialhose API sits behind Cloudflare, which rejects some non-browser User-Agent headers. The SDK defaults to a Chrome 124 UA — don't change it unless you've verified the new UA works. + +**Entity timeline uses cumulative differencing.** The analytics timeline endpoint is campaign-scoped and ignores `content_search`. The SDK facets `/mentions/` by day using cumulative `date_from`-only queries and subtracts consecutive counts. This avoids the API's `date_to` inclusivity bug: overlapping `[date_from, date_to]` windows share a day and double-count it. Don't "simplify" this to day windows. + +**Sentiment reconciliation checks.** The `getEntityStats` exact sentiment and platform distributions validate that facet counts reconcile with the known total (e.g., positive + negative + neutral === total). If they don't match, the API silently dropped the `content_search` filter — the SDK falls back to estimates from the brief's sample rather than showing wrong data. + +**Rate limit ~60 req/min per API key.** Entity analytics fan out many parallel requests. Use the `cache` option with a persistent store (Redis, Next.js Data Cache) to keep warm loads under the limit. + +**Never expose the API key to the browser.** This SDK is designed for server-side use. Always set `SOCIALHOSE_API_KEY` as a server-side environment variable. + ## Errors Failed requests throw `SocialhoseError` with `status`, `path`, and response `body` when available. diff --git a/sdks/javascript/package.json b/sdks/javascript/package.json index e13f7c9..05b1dd0 100644 --- a/sdks/javascript/package.json +++ b/sdks/javascript/package.json @@ -5,7 +5,11 @@ "license": "MIT", "type": "module", "sideEffects": false, - "files": ["dist", "README.md", "LICENSE"], + "files": [ + "dist", + "README.md", + "LICENSE" + ], "main": "./dist/index.cjs", "module": "./dist/index.js", "types": "./dist/index.d.ts", @@ -16,20 +20,30 @@ "require": "./dist/index.cjs" } }, - "engines": { "node": ">=18" }, + "engines": { + "node": ">=18" + }, "scripts": { "build": "tsup src/index.ts --format esm,cjs --dts --sourcemap --clean", "test": "vitest run", "typecheck": "tsc --noEmit", "prepublishOnly": "pnpm test && pnpm typecheck && pnpm build" }, - "keywords": ["socialhose", "social-listening", "social-media", "public-api", "monitoring"], - "publishConfig": { "access": "public", "provenance": true }, + "keywords": [ + "socialhose", + "social-listening", + "social-media", + "public-api", + "monitoring" + ], + "publishConfig": { + "access": "public", + "provenance": true + }, "devDependencies": { "@types/node": "20.16.0", "tsup": "8.3.0", - "typescript": "5.6.2", + "typescript": "^5.6.2", "vitest": "2.1.1" } } - diff --git a/sdks/javascript/src/cache.ts b/sdks/javascript/src/cache.ts new file mode 100644 index 0000000..50f8b04 --- /dev/null +++ b/sdks/javascript/src/cache.ts @@ -0,0 +1,41 @@ +export interface Cache { + get(key: string): Promise; + set(key: string, value: unknown, ttlMs: number): Promise; + delete(key: string): Promise; +} + +type Entry = { at: number; value: unknown; ttlMs: number }; + +/** In-memory Map-backed cache with per-entry TTL. */ +export class MemoryCache implements Cache { + private readonly map = new Map(); + + async get(key: string): Promise { + const entry = this.map.get(key); + if (!entry) return undefined; + if (entry.ttlMs > 0 && Date.now() - entry.at > entry.ttlMs) { + this.map.delete(key); + return undefined; + } + return entry.value; + } + + async set(key: string, value: unknown, ttlMs: number): Promise { + this.map.set(key, { at: Date.now(), value, ttlMs }); + } + + async delete(key: string): Promise { + this.map.delete(key); + } +} + +/** No-op cache — all operations are no-ops; useful for disabling caching. */ +export class NoopCache implements Cache { + async get(_key: string): Promise { + return undefined; + } + + async set(_key: string, _value: unknown, _ttlMs: number): Promise {} + + async delete(_key: string): Promise {} +} diff --git a/sdks/javascript/src/index.ts b/sdks/javascript/src/index.ts index c886498..78790f0 100644 --- a/sdks/javascript/src/index.ts +++ b/sdks/javascript/src/index.ts @@ -1,3 +1,7 @@ +export type { Cache } from './cache'; +export { MemoryCache, NoopCache } from './cache'; +import { type Cache, MemoryCache, NoopCache } from './cache'; + export type Sentiment = 'positive' | 'negative' | 'neutral'; export type SentimentSplit = { positive: number; negative: number; neutral: number }; export type QueryValue = string | number | boolean | null | undefined; @@ -12,10 +16,16 @@ export interface SocialhoseClientOptions { retries?: number; retryDelayMs?: (attempt: number) => number; cacheTtlMs?: number; + /** Inject a custom cache (Redis, Next.js Data Cache, etc.). If provided, cacheTtlMs is ignored for the internal map. */ + cache?: Cache; defaultHeaders?: Record; } export interface RequestOptions { + /** + * Reserved for cache-implementation use; not forwarded to fetch. + * Pass to your Cache.set() implementation to control per-request TTL. + */ revalidateSeconds?: number; signal?: AbortSignal; headers?: Record; @@ -158,6 +168,12 @@ export interface MailingListInvitation { export type InviteOutcome = 'invited' | 'already' | 'error'; +export interface InviteResult { + outcome: InviteOutcome; + invitation?: MailingListInvitation; + detail?: string; +} + export interface Paginated { count: number; next: string | null; @@ -179,6 +195,37 @@ export interface MentionFilters extends AnalyticsFilters { ordering?: string; } +// ---------- Entity-level analytics (content_search faceting) ---------- +// +// The analytics endpoints are campaign-scoped only, but /mentions/ accepts +// content_search composed with sentiment/platform/date filters and always +// returns an exact `count`. We build per-entity views by faceting that search: +// one request gives count + a 20-mention sample (sentiment/platform are exact +// when count <= 20, which covers most entities at current volume); a few +// count-only requests add precise week-over-week momentum. + +export interface PlatformShare { + platform: string; + count: number; +} + +export interface EntityBrief { + term: string; + total: number; + exact: boolean; // sample covers the full population (count <= sample size) + sentiment: SentimentSplit; + platformMix: PlatformShare[]; + sample: Mention[]; // up to 20, ordered by engagement +} + +export interface EntityStats extends EntityBrief { + recent: Mention[]; // up to 20, newest first + recent7d: number; + prev7d: number; + momentumPct: number | null; // last 7 days vs the prior 7 + sparkline: { date: string; count: number }[]; +} + export class SocialhoseError extends Error { readonly status?: number; readonly path: string; @@ -195,14 +242,13 @@ export class SocialhoseError extends Error { } } -type CacheEntry = { at: number; value: unknown }; - -type FetchInitWithNext = RequestInit & { next?: { revalidate?: number } }; - const DEFAULT_BASE_URL = 'https://socialhose.net/api/public/v1'; const DEFAULT_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36'; +// Platform enum the /mentions/ `platforms` filter accepts. +const PLATFORM_KEYS = ['twitter', 'reddit', 'facebook', 'instagram', 'tiktok', 'linkedin'] as const; + function sleep(ms: number): Promise { if (ms <= 0) return Promise.resolve(); return new Promise((resolve) => setTimeout(resolve, ms)); @@ -245,6 +291,27 @@ function timeoutSignal(timeoutMs: number, upstream?: AbortSignal): AbortSignal { return AbortSignal.timeout(timeoutMs); } +/** Returns YYYY-MM-DD for today ± offsetDays (UTC). */ +function isoDay(offsetDays = 0): string { + const d = new Date(); + d.setUTCDate(d.getUTCDate() + offsetDays); + return d.toISOString().slice(0, 10); +} + +function sentimentOf(mentions: Mention[]): SentimentSplit { + const s: SentimentSplit = { positive: 0, negative: 0, neutral: 0 }; + for (const m of mentions) if (m.sentiment in s) s[m.sentiment] += 1; + return s; +} + +function platformMixOf(mentions: Mention[]): PlatformShare[] { + const mix = new Map(); + for (const m of mentions) mix.set(m.platform, (mix.get(m.platform) ?? 0) + 1); + return [...mix.entries()] + .map(([platform, count]) => ({ platform, count })) + .sort((a, b) => b.count - a.count); +} + export class SocialhoseClient { readonly apiKey: string; readonly baseUrl: string; @@ -256,7 +323,8 @@ export class SocialhoseClient { private readonly fetchImpl: typeof fetch; private readonly retryDelayMs: (attempt: number) => number; private readonly defaultHeaders: Record; - private readonly cache = new Map(); + private readonly cacheImpl: Cache; + private readonly usedCacheKeys = new Set(); constructor(options: SocialhoseClientOptions) { if (!options.fetch && typeof fetch === 'undefined') throw new Error('A fetch implementation is required'); @@ -269,20 +337,26 @@ export class SocialhoseClient { this.retryDelayMs = options.retryDelayMs ?? ((attempt) => 400 * 2 ** attempt + Math.random() * 200); this.cacheTtlMs = options.cacheTtlMs ?? 60_000; this.defaultHeaders = options.defaultHeaders ?? {}; + this.cacheImpl = options.cache ?? (this.cacheTtlMs > 0 ? new MemoryCache() : new NoopCache()); } - clearCache(): void { - this.cache.clear(); + async clearCache(): Promise { + const keys = [...this.usedCacheKeys]; + this.usedCacheKeys.clear(); + await Promise.all(keys.map((k) => this.cacheImpl.delete(k))); } async get(path: string, params: QueryParams = {}, options: RequestOptions = {}): Promise { const query = encodeParams(params); const url = `${joinUrl(this.baseUrl, path)}${query ? `?${query}` : ''}`; - const hit = this.cache.get(url); - if (this.cacheTtlMs > 0 && hit && Date.now() - hit.at < this.cacheTtlMs) return hit.value as T; + + const cached = await this.cacheImpl.get(url); + if (cached !== undefined) return cached as T; const value = await this.request('GET', path, url, undefined, options); - if (this.cacheTtlMs > 0) this.cache.set(url, { at: Date.now(), value }); + const ttlMs = options.revalidateSeconds !== undefined ? options.revalidateSeconds * 1000 : this.cacheTtlMs; + await this.cacheImpl.set(url, value, ttlMs); + this.usedCacheKeys.add(url); return value; } @@ -389,7 +463,7 @@ export class SocialhoseClient { listId: string, invite: { email: string; first_name?: string; last_name?: string; invitation_message?: string }, options: RequestOptions = {}, - ): Promise<{ outcome: InviteOutcome; invitation?: MailingListInvitation; detail?: string }> { + ): Promise { const { status, data } = await this.post<{ status?: string; invitation?: MailingListInvitation; @@ -401,6 +475,201 @@ export class SocialhoseClient { return { outcome: 'error', detail: data?.detail ?? `Socialhose returned HTTP ${status}` }; } + // ---------- Entity analytics ---------- + + /** One request: count + a top-engagement sample, with sentiment/platform derived. */ + async getEntityBrief(term: string, campaignId?: string, options: RequestOptions = {}): Promise { + const d = await this.getMentions( + { campaign_ids: campaignId, content_search: term, ordering: '-engagement_count' }, + options, + ); + return { + term, + total: d.count, + exact: d.count <= d.results.length, + sentiment: sentimentOf(d.results), + platformMix: platformMixOf(d.results), + sample: d.results, + }; + } + + /** + * Full single-entity dashboard: brief + exact distributions + newest mentions + daily timeline. + * All sub-requests are best-effort — a single failure never zeros out the entity. + */ + async getEntityStats(term: string, campaignId?: string, options: RequestOptions = {}): Promise { + const base = { campaign_ids: campaignId, content_search: term }; + + const brief = await this.getEntityBrief(term, campaignId, options); + + const [recentPage, sparkline, exactSentiment, exactPlatformMix] = await Promise.all([ + this.getMentions({ ...base, ordering: '-published_at' }, options).catch(() => null), + this.getEntityTimeline(term, campaignId, brief.total, undefined, options).catch( + () => [] as { date: string; count: number }[], + ), + this.getExactSentiment(term, campaignId, brief.total, options).catch(() => null), + this.getExactPlatformMix(term, campaignId, brief.total, options).catch(() => null), + ]); + + const recent = recentPage?.results ?? []; + + const sumOf = (pts: { count: number }[]) => pts.reduce((s, p) => s + p.count, 0); + const recent7d = sumOf(sparkline.slice(-7)); + const prev7d = sumOf(sparkline.slice(-14, -7)); + const momentumPct = + sparkline.length === 0 + ? null + : prev7d > 0 + ? ((recent7d - prev7d) / prev7d) * 100 + : recent7d > 0 + ? 100 + : null; + + const distributionsExact = exactSentiment !== null && exactPlatformMix !== null; + + return { + ...brief, + exact: distributionsExact ? true : brief.exact, + sentiment: exactSentiment ?? brief.sentiment, + platformMix: exactPlatformMix ?? brief.platformMix, + recent, + recent7d, + prev7d, + momentumPct, + sparkline, + }; + } + + /** Fetch briefs for many entities with bounded concurrency (rate-limit friendly). */ + async getEntityBriefs( + terms: string[], + campaignId?: string, + concurrency = 20, + options: RequestOptions = {}, + ): Promise> { + const out = new Map(); + let cursor = 0; + const worker = async () => { + while (cursor < terms.length) { + const term = terms[cursor++]; + try { + out.set(term, await this.getEntityBrief(term, campaignId, options)); + } catch { + // Silent: the entity simply doesn't appear in the map. + } + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, terms.length) }, worker)); + return out; + } + + /** Resolve a live campaign id by matching a substring of its name. */ + async getCampaignIdByMatch(match: string, options: RequestOptions = {}): Promise { + const campaigns = await this.getCampaigns(options).catch(() => [] as Campaign[]); + const needle = match.toLowerCase(); + return campaigns.find((c) => c.name.toLowerCase().includes(needle))?.id; + } + + /** + * Exact sentiment split for a term, by faceting /mentions/ count over each + * sentiment value. Returns null if any facet fails or totals don't reconcile + * with `total` (API silently ignored the filter). + */ + private async getExactSentiment( + term: string, + campaignId: string | undefined, + total: number, + options: RequestOptions = {}, + ): Promise { + const order: Sentiment[] = ['positive', 'negative', 'neutral']; + const counts = await Promise.all( + order.map((s) => + this.getMentions({ campaign_ids: campaignId, content_search: term, sentiments: s }, options) + .then((d) => d.count) + .catch(() => null), + ), + ); + if (counts.some((c) => c === null)) return null; + const [positive, negative, neutral] = counts as number[]; + // Every mention carries exactly one sentiment; facets must sum to the known total. + // A mismatch means the filter wasn't honored — distrust it. + if (positive + negative + neutral !== total) return null; + return { positive, negative, neutral }; + } + + /** + * Exact platform mix for a term, by faceting /mentions/ count over each platform. + * Returns null if any facet fails or the facets sum to more than the known total + * (the `platforms` filter was ignored). A sum below total is fine — mentions may + * sit on platforms outside PLATFORM_KEYS. + */ + private async getExactPlatformMix( + term: string, + campaignId: string | undefined, + total: number, + options: RequestOptions = {}, + ): Promise { + const counts = await Promise.all( + PLATFORM_KEYS.map((platform) => + this.getMentions({ campaign_ids: campaignId, content_search: term, platforms: platform }, options) + .then((d) => ({ platform, count: d.count }) as PlatformShare) + .catch(() => null), + ), + ); + if (counts.some((c) => c === null)) return null; + const mix = counts as PlatformShare[]; + if (mix.reduce((sum, p) => sum + p.count, 0) > total) return null; + return mix.filter((p) => p.count > 0).sort((a, b) => b.count - a.count); + } + + /** + * Real daily mention volume for the last `days` days. + * + * Uses CUMULATIVE DIFFERENCING (date_from only) rather than [date_from, date_to] + * windows. The API counts date_to inclusively, so adjacent windows share a day + * and double-count interior days. Differences of a monotonic cumulative series + * are non-negative and sum to at most `total`. + * + * If the earliest cumulative exceeds `total`, content_search was dropped on + * date-filtered queries — return [] rather than wrong bars. + */ + private async getEntityTimeline( + term: string, + campaignId: string | undefined, + total: number, + days = 14, + options: RequestOptions = {}, + ): Promise<{ date: string; count: number }[]> { + // One boundary per day plus tomorrow (+1) so today differences out cleanly. + const offsets = Array.from({ length: days + 1 }, (_, i) => -(days - 1 - i)); // oldest .. +1 + const cumulative = new Array(offsets.length).fill(null); + let cursor = 0; + const worker = async () => { + while (cursor < offsets.length) { + const idx = cursor++; + const d = await this.getMentions( + { campaign_ids: campaignId, content_search: term, date_from: isoDay(offsets[idx]) }, + options, + ).catch(() => null); + if (d) cumulative[idx] = d.count; + } + }; + // Bounded fan-out: the retry/backoff absorbs the rest of the burst. + await Promise.all(Array.from({ length: Math.min(6, offsets.length) }, worker)); + + const earliest = cumulative[0]; + if (earliest != null && earliest > total) return []; + + const points: { date: string; count: number }[] = []; + for (let i = 0; i < days; i++) { + const hi = cumulative[i]; // on/after day i + const lo = cumulative[i + 1]; // on/after the next day + if (hi == null || lo == null) continue; + points.push({ date: isoDay(offsets[i]), count: Math.max(0, hi - lo) }); + } + return points; + } + private async request( method: 'GET' | 'POST', path: string, @@ -426,7 +695,7 @@ export class SocialhoseClient { for (let attempt = 0; attempt <= this.retries; attempt++) { try { - const init: FetchInitWithNext = { + const init: RequestInit = { method, headers: { ...this.defaultHeaders, @@ -439,9 +708,6 @@ export class SocialhoseClient { body: method === 'POST' ? JSON.stringify(body) : undefined, signal: timeoutSignal(this.timeoutMs, options.signal), }; - if (method === 'GET' && options.revalidateSeconds !== undefined) { - init.next = { revalidate: options.revalidateSeconds }; - } if (method === 'POST') init.cache = 'no-store'; res = await this.fetchImpl(url, init); @@ -482,4 +748,3 @@ export class SocialhoseClient { export function createSocialhoseClient(options: SocialhoseClientOptions): SocialhoseClient { return new SocialhoseClient(options); } - diff --git a/sdks/javascript/test/client.test.ts b/sdks/javascript/test/client.test.ts index 3792bae..1f9966d 100644 --- a/sdks/javascript/test/client.test.ts +++ b/sdks/javascript/test/client.test.ts @@ -1,9 +1,42 @@ import { describe, expect, it, vi } from 'vitest'; -import { SocialhoseClient, SocialhoseError } from '../src/index'; +import { type Cache, MemoryCache, NoopCache, SocialhoseClient, SocialhoseError, type Mention } from '../src/index'; const ok = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); +const paginated = (results: unknown[], count?: number) => + ok({ count: count ?? results.length, next: null, previous: null, results }); + +function makeMention(overrides: Partial = {}): Mention { + return { + id: 'mid', + platform: 'twitter', + campaign_id: 'cid', + campaign_name: 'test', + classification: 'news', + content: 'hello', + title: null, + url: 'https://example.com', + sentiment: 'positive', + sentiment_score: 0.8, + engagement_count: 10, + likes: 5, + shares: 3, + comments: 2, + views_count: 100, + has_media: false, + hashtags: [], + keywords_matched: [], + language: 'en', + country: null, + published_at: '2026-05-29T00:00:00Z', + author: { name: 'Alice', handle: '@alice', avatar_url: null, url: null, followers: 100, verified: false }, + ...overrides, + }; +} + +// ---------- existing tests ---------- + describe('SocialhoseClient', () => { it('sends Api-Key auth, browser-like user-agent, and query params', async () => { const fetchMock = vi.fn(async () => ok({ count: 0, next: null, previous: null, results: [] })); @@ -80,3 +113,381 @@ describe('SocialhoseClient', () => { }); }); +// ---------- MemoryCache ---------- + +describe('MemoryCache', () => { + it('returns undefined for missing key', async () => { + const cache = new MemoryCache(); + expect(await cache.get('k')).toBeUndefined(); + }); + + it('returns stored value within TTL', async () => { + const cache = new MemoryCache(); + await cache.set('k', { data: 42 }, 60_000); + expect(await cache.get('k')).toEqual({ data: 42 }); + }); + + it('returns undefined after TTL expires', async () => { + const cache = new MemoryCache(); + await cache.set('k', 'v', 1); // 1 ms TTL + await new Promise((r) => setTimeout(r, 10)); + expect(await cache.get('k')).toBeUndefined(); + }); + + it('deletes a stored entry', async () => { + const cache = new MemoryCache(); + await cache.set('k', 'v', 60_000); + await cache.delete('k'); + expect(await cache.get('k')).toBeUndefined(); + }); +}); + +// ---------- NoopCache ---------- + +describe('NoopCache', () => { + it('always returns undefined regardless of what was set', async () => { + const cache = new NoopCache(); + await cache.set('k', 'v', 60_000); + expect(await cache.get('k')).toBeUndefined(); + }); +}); + +// ---------- injectable cache ---------- + +describe('injectable cache', () => { + it('uses provided cache instead of internal map', async () => { + const mockCache: Cache = { + get: vi.fn().mockResolvedValue(undefined), + set: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(undefined), + }; + const fetchMock = vi.fn(async () => paginated([])); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cache: mockCache }); + + await client.getMentions({}); + + expect(mockCache.get).toHaveBeenCalled(); + expect(mockCache.set).toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('returns cached value from injected cache without fetching', async () => { + const cached = { count: 99, next: null, previous: null, results: [] }; + const mockCache: Cache = { + get: vi.fn().mockResolvedValue(cached), + set: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(undefined), + }; + const fetchMock = vi.fn(); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cache: mockCache }); + + const result = await client.getMentions({}); + + expect(result).toEqual(cached); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +// ---------- getEntityBrief ---------- + +describe('getEntityBrief', () => { + it('derives sentiment and platform mix from sample', async () => { + const results = [ + makeMention({ sentiment: 'positive', platform: 'twitter' }), + makeMention({ sentiment: 'negative', platform: 'reddit' }), + makeMention({ sentiment: 'positive', platform: 'twitter' }), + ]; + const fetchMock = vi.fn(async () => paginated(results, 3)); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0 }); + + const brief = await client.getEntityBrief('ozempic'); + + expect(brief.term).toBe('ozempic'); + expect(brief.total).toBe(3); + expect(brief.exact).toBe(true); // count (3) <= sample.length (3) + expect(brief.sentiment).toEqual({ positive: 2, negative: 1, neutral: 0 }); + expect(brief.platformMix[0]).toEqual({ platform: 'twitter', count: 2 }); + expect(brief.platformMix[1]).toEqual({ platform: 'reddit', count: 1 }); + expect(brief.sample).toHaveLength(3); + }); + + it('marks exact=false when total exceeds sample length', async () => { + const results = [makeMention()]; + const fetchMock = vi.fn(async () => paginated(results, 500)); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0 }); + + const brief = await client.getEntityBrief('hospital'); + + expect(brief.total).toBe(500); + expect(brief.exact).toBe(false); + }); +}); + +// ---------- getEntityStats best-effort ---------- + +describe('getEntityStats', () => { + it('sub-request failures do not zero out the entity', async () => { + const briefResults = [makeMention({ sentiment: 'positive', platform: 'twitter' })]; + let callCount = 0; + const fetchMock = vi.fn(async () => { + callCount++; + if (callCount === 1) return paginated(briefResults, 1); + return new Response('error', { status: 502 }); + }); + const client = new SocialhoseClient({ + apiKey: 'key', + fetch: fetchMock, + cacheTtlMs: 0, + retries: 0, + }); + + const stats = await client.getEntityStats('ozempic'); + + // Brief data is always present + expect(stats.term).toBe('ozempic'); + expect(stats.total).toBe(1); + // Falls back to brief's sample sentiment + expect(stats.sentiment).toEqual({ positive: 1, negative: 0, neutral: 0 }); + // Sub-requests failed: recent is empty, sparkline is empty + expect(stats.recent).toEqual([]); + expect(stats.sparkline).toEqual([]); + expect(stats.momentumPct).toBeNull(); + }); + + it('uses exact sentiment when facets reconcile with total', async () => { + const briefResults = [ + makeMention({ sentiment: 'positive' }), + makeMention({ sentiment: 'negative' }), + ]; + const fetchMock = vi.fn(async (url) => { + if ((url as string).includes('ordering=-engagement_count')) return paginated(briefResults, 2); + if ((url as string).includes('ordering=-published_at')) return paginated([], 2); + if ((url as string).includes('date_from=')) return paginated([], 0); + if ((url as string).includes('sentiments=positive')) return paginated([], 1); + if ((url as string).includes('sentiments=negative')) return paginated([], 1); + if ((url as string).includes('sentiments=neutral')) return paginated([], 0); + if ((url as string).includes('platforms=')) return paginated([], 0); + return paginated([], 0); + }); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + + const stats = await client.getEntityStats('flu'); + + // Exact split: 1+1+0=2 == total(2) → use exact + expect(stats.sentiment).toEqual({ positive: 1, negative: 1, neutral: 0 }); + expect(stats.exact).toBe(true); + }); + + it('falls back to brief sentiment when facets do not reconcile', async () => { + const briefResults = [ + makeMention({ sentiment: 'positive' }), + makeMention({ sentiment: 'positive' }), + ]; + const fetchMock = vi.fn(async (url) => { + if ((url as string).includes('ordering=-engagement_count')) return paginated(briefResults, 2); + if ((url as string).includes('ordering=-published_at')) return paginated([], 2); + if ((url as string).includes('date_from=')) return paginated([], 0); + // Facets sum to 5, total is 2 → mismatch + if ((url as string).includes('sentiments=positive')) return paginated([], 3); + if ((url as string).includes('sentiments=negative')) return paginated([], 1); + if ((url as string).includes('sentiments=neutral')) return paginated([], 1); + if ((url as string).includes('platforms=')) return paginated([], 0); + return paginated([], 0); + }); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + + const stats = await client.getEntityStats('flu'); + + // Facets don't reconcile → fall back to brief's sample sentiment + expect(stats.sentiment).toEqual({ positive: 2, negative: 0, neutral: 0 }); + }); +}); + +// ---------- exact sentiment reconciliation ---------- + +describe('getExactSentiment (via private access)', () => { + it('returns null when facets sum exceeds total', async () => { + const fetchMock = vi.fn() + .mockResolvedValueOnce(paginated([], 6)) // positive + .mockResolvedValueOnce(paginated([], 3)) // negative + .mockResolvedValueOnce(paginated([], 4)); // neutral — sum=13 ≠ 10 + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = await (client as any).getExactSentiment('term', undefined, 10); + expect(result).toBeNull(); + }); + + it('returns the split when facets sum exactly equals total', async () => { + const fetchMock = vi.fn() + .mockResolvedValueOnce(paginated([], 4)) // positive + .mockResolvedValueOnce(paginated([], 3)) // negative + .mockResolvedValueOnce(paginated([], 3)); // neutral — sum=10 == 10 + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = await (client as any).getExactSentiment('term', undefined, 10); + expect(result).toEqual({ positive: 4, negative: 3, neutral: 3 }); + }); + + it('returns null when a facet request fails', async () => { + const fetchMock = vi.fn() + .mockResolvedValueOnce(paginated([], 4)) + .mockResolvedValueOnce(new Response('error', { status: 500 })) + .mockResolvedValueOnce(paginated([], 3)); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const result = await (client as any).getExactSentiment('term', undefined, 10); + expect(result).toBeNull(); + }); +}); + +// ---------- cumulative timeline differencing ---------- + +describe('getEntityTimeline (via private access)', () => { + it('produces correct daily counts via cumulative differencing', async () => { + // days=3 → offsets = [-2, -1, 0, 1], 4 boundary requests + const today = new Date(); + const computeDay = (offset: number) => { + const d = new Date(today); + d.setUTCDate(d.getUTCDate() + offset); + return d.toISOString().slice(0, 10); + }; + + const cumCounts: Record = { + [computeDay(-2)]: 30, + [computeDay(-1)]: 20, + [computeDay(0)]: 5, + [computeDay(1)]: 0, + }; + + const fetchMock = vi.fn(async (url) => { + const dateFrom = new URL(url).searchParams.get('date_from') ?? ''; + return paginated([], cumCounts[dateFrom] ?? 0); + }); + + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const timeline = await (client as any).getEntityTimeline('term', undefined, 100, 3); + + expect(timeline).toHaveLength(3); + // day -2: 30 - 20 = 10 + // day -1: 20 - 5 = 15 + // day 0: 5 - 0 = 5 + const find = (offset: number) => + (timeline as { date: string; count: number }[]).find((p) => p.date === computeDay(offset)); + expect(find(-2)?.count).toBe(10); + expect(find(-1)?.count).toBe(15); + expect(find(0)?.count).toBe(5); + }); + + it('returns [] when earliest cumulative exceeds total (filter was ignored)', async () => { + const today = new Date(); + const computeDay = (offset: number) => { + const d = new Date(today); + d.setUTCDate(d.getUTCDate() + offset); + return d.toISOString().slice(0, 10); + }; + + // Earliest boundary returns 200, total is 50 → filter was ignored + const fetchMock = vi.fn(async (url) => { + const dateFrom = new URL(url).searchParams.get('date_from'); + const count = dateFrom === computeDay(-2) ? 200 : 50; + return paginated([], count); + }); + + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const timeline = await (client as any).getEntityTimeline('term', undefined, 50, 3); + + expect(timeline).toEqual([]); + }); +}); + +// ---------- getCampaignIdByMatch ---------- + +describe('getCampaignIdByMatch', () => { + it('returns the id of the first campaign whose name contains the match', async () => { + const campaigns = [ + { id: 'c1', name: 'Sudan Watch', description: '', status: 'active', campaign_type: 'monitoring', platforms: [], tags: [] }, + { id: 'c2', name: 'Health Monitor', description: '', status: 'active', campaign_type: 'monitoring', platforms: [], tags: [] }, + ]; + const fetchMock = vi.fn(async () => paginated(campaigns, 2)); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0 }); + + expect(await client.getCampaignIdByMatch('health')).toBe('c2'); + expect(await client.getCampaignIdByMatch('sudan')).toBe('c1'); + }); + + it('returns undefined when no campaign matches', async () => { + const campaigns = [ + { id: 'c1', name: 'Sudan Watch', description: '', status: 'active', campaign_type: 'monitoring', platforms: [], tags: [] }, + ]; + const fetchMock = vi.fn(async () => paginated(campaigns, 1)); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0 }); + + expect(await client.getCampaignIdByMatch('nonexistent')).toBeUndefined(); + }); + + it('returns undefined when getCampaigns fails', async () => { + const fetchMock = vi.fn(async () => new Response('error', { status: 500 })); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + + expect(await client.getCampaignIdByMatch('health')).toBeUndefined(); + }); +}); + +// ---------- getEntityBriefs bounded concurrency ---------- + +describe('getEntityBriefs', () => { + it('processes all terms and returns a map', async () => { + const fetchMock = vi.fn(async () => paginated([makeMention()], 1)); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + + const result = await client.getEntityBriefs(['a', 'b', 'c', 'd', 'e']); + + expect(result.size).toBe(5); + for (const t of ['a', 'b', 'c', 'd', 'e']) expect(result.has(t)).toBe(true); + }); + + it('silently skips terms that fail', async () => { + let callCount = 0; + const fetchMock = vi.fn(async () => { + callCount++; + if (callCount === 2) return new Response('error', { status: 500 }); + return paginated([makeMention()], 1); + }); + // concurrency=1 makes order deterministic: a, b (fails), c + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + + const result = await client.getEntityBriefs(['a', 'b', 'c'], undefined, 1); + + expect(result.size).toBe(2); + expect(result.has('a')).toBe(true); + expect(result.has('b')).toBe(false); + expect(result.has('c')).toBe(true); + }); + + it('respects bounded concurrency', async () => { + let inFlight = 0; + let maxInFlight = 0; + + const fetchMock = vi.fn(async () => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + // Yield to allow other workers to start + await new Promise((r) => setTimeout(r, 5)); + inFlight--; + return paginated([makeMention()], 1); + }); + + const terms = Array.from({ length: 10 }, (_, i) => `term${i}`); + const client = new SocialhoseClient({ apiKey: 'key', fetch: fetchMock, cacheTtlMs: 0, retries: 0 }); + + await client.getEntityBriefs(terms, undefined, 3); + + expect(maxInFlight).toBeLessThanOrEqual(3); + expect(fetchMock).toHaveBeenCalledTimes(10); + }); +});