From 88014118669db1d44bc69046c0c706c785931276 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Tue, 7 Jul 2026 12:56:19 +0200 Subject: [PATCH 1/2] Handle C++ raw strings. --- .../commentfilter/c_style_filter_test.go | 25 +++++++ embedding/commentfilter/config.go | 34 ++++++++-- .../commentfilter/marker_comment_filter.go | 66 +++++++++++++++++++ embedding/commentfilter/marker_syntax.go | 3 + 4 files changed, 121 insertions(+), 7 deletions(-) diff --git a/embedding/commentfilter/c_style_filter_test.go b/embedding/commentfilter/c_style_filter_test.go index b057f7b..900d343 100644 --- a/embedding/commentfilter/c_style_filter_test.go +++ b/embedding/commentfilter/c_style_filter_test.go @@ -100,4 +100,29 @@ var _ = Describe("C and C++", func() { assertFiltered("sample.cpp", RetainNone, lines, expected) }) + + It("should strip comments without treating raw string text as comments", func() { + lines := []string{ + "// header comment", + `const char* single = R"(Keep // and /* markers */ in raw text)";`, + `const char* tagged = R"tag(Keep )" and // markers)tag"; // inline comment`, + `const char* multi = R"(`, + `Keep // marker`, + `Keep /* marker */`, + `)"; /* trailing block */`, + "int value = 1; // inline comment", + } + + expected := []string{ + `const char* single = R"(Keep // and /* markers */ in raw text)";`, + `const char* tagged = R"tag(Keep )" and // markers)tag"; `, + `const char* multi = R"(`, + `Keep // marker`, + `Keep /* marker */`, + `)"; `, + "int value = 1; ", + } + + assertFiltered("sample.cpp", RetainNone, lines, expected) + }) }) diff --git a/embedding/commentfilter/config.go b/embedding/commentfilter/config.go index 593f451..d77e741 100644 --- a/embedding/commentfilter/config.go +++ b/embedding/commentfilter/config.go @@ -30,6 +30,11 @@ const ( cStyleBlockCommentStart = "/*" cStyleBlockCommentEnd = "*/" cStyleDocCommentStart = "/**" + cPlusPlusRawString = "R" + cPlusPlusURawString = "uR" + cPlusPlusU8RawString = "u8R" + cPlusPlusURawStringWide = "UR" + cPlusPlusLRawString = "LR" javaTextBlockDelimiter = "\"\"\"" pythonDoubleQuoteBlock = "\"\"\"" pythonSingleQuoteBlock = "'''" @@ -50,13 +55,13 @@ var filtersByExtension = map[string]filterEntry{ // C/C++ ".c": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), - ".h": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), - ".cc": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), - ".cpp": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), - ".cxx": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), - ".hh": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), - ".hpp": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), - ".hxx": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), + ".h": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes), + ".cc": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes), + ".cpp": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes), + ".cxx": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes), + ".hh": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes), + ".hpp": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes), + ".hxx": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes), // JavaScript ".js": filterConfig(JavaScriptCommentFilter{}, allModes), @@ -124,6 +129,21 @@ var cStyleSyntax = CommentMarker{ QuoteChars: "\"'", } +var cPlusPlusSyntax = CommentMarker{ + Inline: []string{"//"}, + Block: []BlockMarker{ + {Start: cStyleBlockCommentStart, End: cStyleBlockCommentEnd}, + }, + RawStringPrefixes: []string{ + cPlusPlusU8RawString, + cPlusPlusURawStringWide, + cPlusPlusLRawString, + cPlusPlusURawString, + cPlusPlusRawString, + }, + QuoteChars: "\"'", +} + var goSyntax = CommentMarker{ Inline: []string{"//"}, Block: []BlockMarker{ diff --git a/embedding/commentfilter/marker_comment_filter.go b/embedding/commentfilter/marker_comment_filter.go index 15f942a..46bcbb5 100644 --- a/embedding/commentfilter/marker_comment_filter.go +++ b/embedding/commentfilter/marker_comment_filter.go @@ -88,6 +88,9 @@ func (f *markerLineFilter) filterLine() (string, bool) { if f.consumeActiveSegment() { continue } + if f.consumeRawStringStart() { + continue + } if f.consumeTextBlockStart() { continue } @@ -107,6 +110,21 @@ func (f *markerLineFilter) filterLine() (string, bool) { return f.result.String(), f.hadComment } +// consumeRawStringStart starts a raw string literal with a dynamic closing marker. +func (f *markerLineFilter) consumeRawStringStart() bool { + rawString, found := rawStringAt(f.line, f.position, f.filter.Syntax.RawStringPrefixes) + if !found { + return false + } + f.consumeMarker(rawString.open) + f.state.segment = &activeSegment{ + end: rawString.end, + keep: true, + } + + return true +} + // consumeActiveSegment consumes text while the scanner is inside a multi-line construct. func (f *markerLineFilter) consumeActiveSegment() bool { segment := f.state.segment @@ -222,6 +240,54 @@ func prefixAt(line string, position int, prefixes []string) bool { return false } +// rawStringStart describes an opened raw string literal. +type rawStringStart struct { + // open is the complete opening marker copied to output. + open string + + // end is the dynamic marker that closes the literal. + end string +} + +// rawStringAt reports whether a raw string starts at position. +func rawStringAt(line string, position int, prefixes []string) (rawStringStart, bool) { + for _, prefix := range prefixes { + open, end, found := rawStringStartAt(line, position, prefix) + if found { + return rawStringStart{open: open, end: end}, true + } + } + + return rawStringStart{}, false +} + +// rawStringStartAt returns raw string markers for a specific prefix. +func rawStringStartAt(line string, position int, prefix string) (string, string, bool) { + marker := prefix + `"` + if !strings.HasPrefix(line[position:], marker) { + return "", "", false + } + delimiterStart := position + len(marker) + delimiterEnd := delimiterStart + for delimiterEnd < len(line) && line[delimiterEnd] != '(' { + if !validRawStringDelimiterByte(line[delimiterEnd]) { + return "", "", false + } + delimiterEnd++ + } + if delimiterEnd >= len(line) { + return "", "", false + } + delimiter := line[delimiterStart:delimiterEnd] + + return line[position : delimiterEnd+1], ")" + delimiter + `"`, true +} + +// validRawStringDelimiterByte reports whether a byte is valid in a raw string delimiter. +func validRawStringDelimiterByte(char byte) bool { + return char > ' ' && char != '(' && char != ')' && char != '\\' +} + // textBlockAt reports whether one of the given text block markers starts at the position. func textBlockAt(line string, position int, markers []TextBlockMarker) (TextBlockMarker, bool) { for _, marker := range markers { diff --git a/embedding/commentfilter/marker_syntax.go b/embedding/commentfilter/marker_syntax.go index f73022f..cc18177 100644 --- a/embedding/commentfilter/marker_syntax.go +++ b/embedding/commentfilter/marker_syntax.go @@ -67,6 +67,9 @@ type CommentMarker struct { // TextBlocks contains markers that open and close multi-line text literals. TextBlocks []TextBlockMarker + // RawStringPrefixes contains prefixes that open raw string literals with dynamic delimiters. + RawStringPrefixes []string + // QuoteChars contains characters that open and close quoted strings. QuoteChars string } From 388262f831fd4e2976dcc915ba380b4dad07ec28 Mon Sep 17 00:00:00 2001 From: Vladyslav Kuksiuk Date: Tue, 7 Jul 2026 13:27:42 +0200 Subject: [PATCH 2/2] Fix raw prefixes handling. --- embedding/commentfilter/c_style_filter_test.go | 14 ++++++++++++++ embedding/commentfilter/marker_comment_filter.go | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/embedding/commentfilter/c_style_filter_test.go b/embedding/commentfilter/c_style_filter_test.go index 900d343..850d152 100644 --- a/embedding/commentfilter/c_style_filter_test.go +++ b/embedding/commentfilter/c_style_filter_test.go @@ -125,4 +125,18 @@ var _ = Describe("C and C++", func() { assertFiltered("sample.cpp", RetainNone, lines, expected) }) + + It("should not treat raw string prefixes inside identifiers as raw strings", func() { + lines := []string{ + `SOME_MACRO(BAR"abc(x)")`, + "int value = 1; // real comment", + } + + expected := []string{ + `SOME_MACRO(BAR"abc(x)")`, + "int value = 1; ", + } + + assertFiltered("sample.cpp", RetainNone, lines, expected) + }) }) diff --git a/embedding/commentfilter/marker_comment_filter.go b/embedding/commentfilter/marker_comment_filter.go index 46bcbb5..abd9b21 100644 --- a/embedding/commentfilter/marker_comment_filter.go +++ b/embedding/commentfilter/marker_comment_filter.go @@ -264,6 +264,9 @@ func rawStringAt(line string, position int, prefixes []string) (rawStringStart, // rawStringStartAt returns raw string markers for a specific prefix. func rawStringStartAt(line string, position int, prefix string) (string, string, bool) { marker := prefix + `"` + if position > 0 && rawStringIdentifierByte(line[position-1]) { + return "", "", false + } if !strings.HasPrefix(line[position:], marker) { return "", "", false } @@ -288,6 +291,14 @@ func validRawStringDelimiterByte(char byte) bool { return char > ' ' && char != '(' && char != ')' && char != '\\' } +// rawStringIdentifierByte reports whether a byte can continue a C/C++ identifier. +func rawStringIdentifierByte(char byte) bool { + return (char >= 'A' && char <= 'Z') || + (char >= 'a' && char <= 'z') || + (char >= '0' && char <= '9') || + char == '_' +} + // textBlockAt reports whether one of the given text block markers starts at the position. func textBlockAt(line string, position int, markers []TextBlockMarker) (TextBlockMarker, bool) { for _, marker := range markers {