Improve C++ comments filtering

embedding/commentfilter/c_style_filter_test.go

@@ -100,4 +100,43 @@ var _ = Describe("C and C++", func() {
 
 		assertFiltered("sample.cpp", RetainNone, lines, expected)
 	})
+
+	It("should strip comments without treating raw string text as comments", func() {
+		lines := []string{
+			"// header comment",
+			`const char* single = R"(Keep // and /* markers */ in raw text)";`,
+			`const char* tagged = R"tag(Keep )" and // markers)tag"; // inline comment`,
+			`const char* multi = R"(`,
+			`Keep // marker`,
+			`Keep /* marker */`,
+			`)"; /* trailing block */`,
+			"int value = 1; // inline comment",
+		}
+
+		expected := []string{
+			`const char* single = R"(Keep // and /* markers */ in raw text)";`,
+			`const char* tagged = R"tag(Keep )" and // markers)tag"; `,
+			`const char* multi = R"(`,
+			`Keep // marker`,
+			`Keep /* marker */`,
+			`)"; `,
+			"int value = 1; ",
+		}
+
+		assertFiltered("sample.cpp", RetainNone, lines, expected)
+	})
+
+	It("should not treat raw string prefixes inside identifiers as raw strings", func() {
+		lines := []string{
+			`SOME_MACRO(BAR"abc(x)")`,
+			"int value = 1; // real comment",
+		}
+
+		expected := []string{
+			`SOME_MACRO(BAR"abc(x)")`,
+			"int value = 1; ",
+		}
+
+		assertFiltered("sample.cpp", RetainNone, lines, expected)
+	})
 })

embedding/commentfilter/config.go

@@ -30,6 +30,11 @@ const (
 	cStyleBlockCommentStart = "/*"
 	cStyleBlockCommentEnd   = "*/"
 	cStyleDocCommentStart   = "/**"
+	cPlusPlusRawString      = "R"
+	cPlusPlusURawString     = "uR"
+	cPlusPlusU8RawString    = "u8R"
+	cPlusPlusURawStringWide = "UR"
+	cPlusPlusLRawString     = "LR"
 	javaTextBlockDelimiter  = "\"\"\""
 	pythonDoubleQuoteBlock  = "\"\"\""
 	pythonSingleQuoteBlock  = "'''"
@@ -50,13 +55,13 @@ var filtersByExtension = map[string]filterEntry{
 
 	// C/C++
 	".c":   filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes),
-	".h":   filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes),
-	".cc":  filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes),
-	".cpp": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes),
-	".cxx": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes),
-	".hh":  filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes),
-	".hpp": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes),
-	".hxx": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes),
+	".h":   filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes),
+	".cc":  filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes),
+	".cpp": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes),
+	".cxx": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes),
+	".hh":  filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes),
+	".hpp": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes),
+	".hxx": filterConfig(MarkerCommentFilter{Syntax: cPlusPlusSyntax}, regularModes),
 
 	// JavaScript
 	".js":  filterConfig(JavaScriptCommentFilter{}, allModes),
@@ -124,6 +129,21 @@ var cStyleSyntax = CommentMarker{
 	QuoteChars: "\"'",
 }
 
+var cPlusPlusSyntax = CommentMarker{
+	Inline: []string{"//"},
+	Block: []BlockMarker{
+		{Start: cStyleBlockCommentStart, End: cStyleBlockCommentEnd},
+	},
+	RawStringPrefixes: []string{
+		cPlusPlusU8RawString,
+		cPlusPlusURawStringWide,
+		cPlusPlusLRawString,
+		cPlusPlusURawString,
+		cPlusPlusRawString,
+	},
+	QuoteChars: "\"'",
+}
+
 var goSyntax = CommentMarker{
 	Inline: []string{"//"},
 	Block: []BlockMarker{

embedding/commentfilter/marker_comment_filter.go

@@ -88,6 +88,9 @@ func (f *markerLineFilter) filterLine() (string, bool) {
 		if f.consumeActiveSegment() {
 			continue
 		}
+		if f.consumeRawStringStart() {
+			continue
+		}
 		if f.consumeTextBlockStart() {
 			continue
 		}
@@ -107,6 +110,21 @@ func (f *markerLineFilter) filterLine() (string, bool) {
 	return f.result.String(), f.hadComment
 }
 
+// consumeRawStringStart starts a raw string literal with a dynamic closing marker.
+func (f *markerLineFilter) consumeRawStringStart() bool {
+	rawString, found := rawStringAt(f.line, f.position, f.filter.Syntax.RawStringPrefixes)
+	if !found {
+		return false
+	}
+	f.consumeMarker(rawString.open)
+	f.state.segment = &activeSegment{
+		end:  rawString.end,
+		keep: true,
+	}
+
+	return true
+}
+
 // consumeActiveSegment consumes text while the scanner is inside a multi-line construct.
 func (f *markerLineFilter) consumeActiveSegment() bool {
 	segment := f.state.segment
@@ -222,6 +240,65 @@ func prefixAt(line string, position int, prefixes []string) bool {
 	return false
 }
 
+// rawStringStart describes an opened raw string literal.
+type rawStringStart struct {
+	// open is the complete opening marker copied to output.
+	open string
+
+	// end is the dynamic marker that closes the literal.
+	end string
+}
+
+// rawStringAt reports whether a raw string starts at position.
+func rawStringAt(line string, position int, prefixes []string) (rawStringStart, bool) {
+	for _, prefix := range prefixes {
+		open, end, found := rawStringStartAt(line, position, prefix)
+		if found {
+			return rawStringStart{open: open, end: end}, true
+		}
+	}
+
+	return rawStringStart{}, false
+}
+
+// rawStringStartAt returns raw string markers for a specific prefix.
+func rawStringStartAt(line string, position int, prefix string) (string, string, bool) {
+	marker := prefix + `"`
+	if position > 0 && rawStringIdentifierByte(line[position-1]) {
+		return "", "", false
+	}
+	if !strings.HasPrefix(line[position:], marker) {
+		return "", "", false
+	}
+	delimiterStart := position + len(marker)
+	delimiterEnd := delimiterStart
+	for delimiterEnd < len(line) && line[delimiterEnd] != '(' {
+		if !validRawStringDelimiterByte(line[delimiterEnd]) {
+			return "", "", false
+		}
+		delimiterEnd++
+	}
+	if delimiterEnd >= len(line) {
+		return "", "", false
+	}
+	delimiter := line[delimiterStart:delimiterEnd]
+
+	return line[position : delimiterEnd+1], ")" + delimiter + `"`, true
+}
+
+// validRawStringDelimiterByte reports whether a byte is valid in a raw string delimiter.
+func validRawStringDelimiterByte(char byte) bool {
+	return char > ' ' && char != '(' && char != ')' && char != '\\'
+}
+
+// rawStringIdentifierByte reports whether a byte can continue a C/C++ identifier.
+func rawStringIdentifierByte(char byte) bool {
+	return (char >= 'A' && char <= 'Z') ||
+		(char >= 'a' && char <= 'z') ||
+		(char >= '0' && char <= '9') ||
+		char == '_'
+}
+
 // textBlockAt reports whether one of the given text block markers starts at the position.
 func textBlockAt(line string, position int, markers []TextBlockMarker) (TextBlockMarker, bool) {
 	for _, marker := range markers {

embedding/commentfilter/marker_syntax.go

@@ -67,6 +67,9 @@ type CommentMarker struct {
 	// TextBlocks contains markers that open and close multi-line text literals.
 	TextBlocks []TextBlockMarker
 
+	// RawStringPrefixes contains prefixes that open raw string literals with dynamic delimiters.
+	RawStringPrefixes []string
+
 	// QuoteChars contains characters that open and close quoted strings.
 	QuoteChars string
 }