From 128e3d8cf4a8bb2ef91a86babf857f259b0321bf Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 10 Oct 2026 23:13:18 +0200 Subject: [PATCH] fix(rst): a section's text no longer reads an unwritten byte rst_body cut a reST section's text at 500 bytes by setting a stop value past the end, then backing off from out[RST_BODY_MAX] while that byte looked like a UTF-8 continuation byte. That byte was never written, so where the text ended (499 or 500 bytes) depended on what the arena memory held, and when a multibyte character did not fit, bytes never written could end up in the text. Only whole characters are written now, and the text ends at the last one written. The same django tree indexed twice gave doc_link_candidates rows that differed in about 250 of 11,300 (through the section text's TF-IDF); with this change two runs are identical, and so are the 6,594 sections. The new test pins the cut at the boundary: a word that fills the 500 bytes, a two-byte character that would cross them, a word cut at a character. Signed-off-by: Martin Vogel --- internal/cbm/doclink_rst.c | 15 +++++---------- tests/test_doc_links_rst.c | 39 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/internal/cbm/doclink_rst.c b/internal/cbm/doclink_rst.c index a50498f14..9169a38d2 100644 --- a/internal/cbm/doclink_rst.c +++ b/internal/cbm/doclink_rst.c @@ -1606,18 +1606,19 @@ static const char *rst_body(rst_doc_t *d, int from, int to) { return NULL; } int w = 0; - for (int k = from; k < to && w < RST_BODY_MAX; k++) { + bool full = false; /* the next character did not fit: the text ends before it */ + for (int k = from; k < to && !full; k++) { const char *s = d->L[k].s; int len = d->L[k].len; int i = 0; - while (i < len && w < RST_BODY_MAX) { + while (i < len && !full) { int n; uint32_t cp = rst_cp(s, len, i, &n); if (rst_space_cp(cp)) { i += n; continue; } - if (w > 0) { + if (w > 0 && w < RST_BODY_MAX) { out[w++] = ' '; } while (i < len) { @@ -1626,7 +1627,7 @@ static const char *rst_body(rst_doc_t *d, int from, int to) { break; } if (w + n > RST_BODY_MAX) { - w = RST_BODY_MAX + 1; /* stop: the next character does not fit */ + full = true; /* only whole characters are written: none is cut */ break; } memcpy(out + w, s + i, (size_t)n); @@ -1635,12 +1636,6 @@ static const char *rst_body(rst_doc_t *d, int from, int to) { } } } - if (w > RST_BODY_MAX) { - w = RST_BODY_MAX; - while (w > 0 && ((unsigned char)out[w] & 0xC0) == 0x80) { - w--; - } - } while (w > 0 && out[w - 1] == ' ') { w--; } diff --git a/tests/test_doc_links_rst.c b/tests/test_doc_links_rst.c index 9694db383..5f3f5a406 100644 --- a/tests/test_doc_links_rst.c +++ b/tests/test_doc_links_rst.c @@ -558,9 +558,48 @@ TEST(rst_ship_gate) { PASS(); } +/* A section's text is cut before the first character that does not fit its + * 500 bytes: only whole characters are written, and the text ends at the last + * one written (it once tested an unwritten byte past the end, so where the + * text ended depended on what that memory held). */ +TEST(rst_section_text_cut) { + static const struct { + const char *tail; + const char *want_tail; + } cases[] = { + {" xxxxx tail", " xxxxx"}, /* a word that ends at byte 500 */ + {" xxxx\xc3\xa9 more", " xxxx"}, /* a two-byte character that would cross it */ + {" yyyyyyyy", " yyyyy"}, /* a word cut at a character */ + }; + for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) { + char doc[1024]; + char want[600]; + int n = snprintf(doc, sizeof(doc), "Title\n=====\n\n"); + int w = 0; + for (int k = 0; k < 99; k++) { /* 99 words: 494 bytes, nine per paragraph */ + n += snprintf(doc + n, sizeof(doc) - (size_t)n, "%sabcd", + k == 0 ? "" + : k % 9 == 0 ? "\n\n" + : " "); + w += snprintf(want + w, sizeof(want) - (size_t)w, "%sabcd", k ? " " : ""); + } + snprintf(doc + n, sizeof(doc) - (size_t)n, "%s\n", cases[c].tail); + snprintf(want + w, sizeof(want) - (size_t)w, "%s", cases[c].want_tail); + CBMFileResult *r = dm_extract(doc, CBM_LANG_RST, "docs/cut.rst"); + ASSERT_NOT_NULL(r); + const CBMDefinition *t = rst_def(r, "Section", "Title"); + ASSERT_NOT_NULL(t); + ASSERT_NOT_NULL(t->docstring); + ASSERT_STR_EQ(t->docstring, want); + cbm_free_result(r); + } + PASS(); +} + SUITE(doc_links_rst) { dm_ship_held_families(); RUN_TEST(rst_scan_structure); + RUN_TEST(rst_section_text_cut); RUN_TEST(rst_python_scope_blob); RUN_TEST(rst_links_pipeline); RUN_TEST(rst_c_member_owner);