diff --git a/embedding/commentfilter/c_style_filter_test.go b/embedding/commentfilter/c_style_filter_test.go index d188443..b38f820 100644 --- a/embedding/commentfilter/c_style_filter_test.go +++ b/embedding/commentfilter/c_style_filter_test.go @@ -80,4 +80,16 @@ var _ = Describe("C and C++", func() { assertFiltered("sample.hpp", RetainBlock, lines, expected) }) + + It("should not close block comments on an overlapping end marker", func() { + lines := []string{ + "int before = 1; /*/ still comment */ int after = 2;", + } + + expected := []string{ + "int before = 1; int after = 2;", + } + + assertFiltered("sample.cpp", RetainNone, lines, expected) + }) }) diff --git a/embedding/commentfilter/config.go b/embedding/commentfilter/config.go index 906f4c5..ececc64 100644 --- a/embedding/commentfilter/config.go +++ b/embedding/commentfilter/config.go @@ -27,7 +27,6 @@ const ( pythonSingleQuoteBlock = "'''" goRawStringDelimiter = "`" goQuoteChars = "\"'" - jsQuoteChars = "\"'`" ) // filtersByExtension is a mapping of the file extension to its comment filter. @@ -64,9 +63,9 @@ var filtersByExtension = map[string]filterEntry{ ".proto": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes), // Python - ".py": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode), - ".pyi": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode), - ".pyw": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode), + ".py": filterConfig(PythonCommentFilter{}, noneMode), + ".pyi": filterConfig(PythonCommentFilter{}, noneMode), + ".pyw": filterConfig(PythonCommentFilter{}, noneMode), // YAML ".yml": filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode), @@ -133,15 +132,6 @@ var hashLineSyntax = CommentMarker{ QuoteChars: "\"'", } -var pythonSyntax = CommentMarker{ - Inline: []string{"#"}, - TextBlocks: []TextBlockMarker{ - {Delimiter: pythonDoubleQuoteBlock, Escapes: true}, - {Delimiter: pythonSingleQuoteBlock, Escapes: true}, - }, - QuoteChars: "\"'", -} - var xmlSyntax = CommentMarker{ Block: []BlockMarker{ {Start: ""}, diff --git a/embedding/commentfilter/csharp_filter.go b/embedding/commentfilter/csharp_filter.go index 0ec705c..70e5c63 100644 --- a/embedding/commentfilter/csharp_filter.go +++ b/embedding/commentfilter/csharp_filter.go @@ -25,21 +25,44 @@ const ( csharpVerbatimInterpolatedStringStart = `@$"` csharpVerbatimStringStart = `@"` csharpInterpolatedStringStart = `$"` + csharpInterpolatedRawStringStart = `$"""` + csharpRawStringDelimiter = `"""` csharpEscapedQuote = `""` csharpEscapedOpenBrace = `{{` csharpEscapedCloseBrace = `}}` + csharpDocLineComment = `///` ) +// csharpInterpolation describes the C# `{...}` string interpolation form. +// Holes open with a bare `{` inside interpolated string text. +var csharpInterpolation = interpolationForm{ + start: "{", + openBrace: '{', + closeBrace: '}', +} + +// csharpStringStart describes a C# string opening token. +type csharpStringStart struct { + // token is the source text that opens the string. + token string + + // verbatim reports whether the string uses verbatim escaping. + verbatim bool + + // interpolated reports whether the string contains interpolation holes. + interpolated bool + + // rawDelimiter closes a raw string; empty for regular and verbatim strings. + rawDelimiter string +} + // CSharpCommentFilter filters C# comments while preserving string literal text. type CSharpCommentFilter struct{} // csharpState tracks C# lexical state that can span source lines. type csharpState struct { - // blockActive reports whether scanning is inside a block comment. - blockActive bool - - // blockKeep reports whether the active block comment should be retained. - blockKeep bool + // block tracks a block comment across source lines. + block blockCommentState // stringActive reports whether scanning is inside a string. stringActive bool @@ -50,6 +73,9 @@ type csharpState struct { // stringInterpolated reports whether the active string has interpolation holes. stringInterpolated bool + // stringRawDelimiter closes an active raw string. + stringRawDelimiter string + // interpolationDepth is the active brace depth inside an interpolation expression. interpolationDepth int @@ -59,23 +85,11 @@ type csharpState struct { // csharpLineFilter filters one C# source line. type csharpLineFilter struct { - // line is the source line being filtered. - line string - - // mode selects which comments to retain. - mode Mode + // lineFilter provides shared line scanning state and helpers. + lineFilter // state tracks C# constructs across lines. state *csharpState - - // result accumulates the filtered source line. - result strings.Builder - - // position is the current byte index in line. - position int - - // hadComment reports whether the line contained a recognized comment. - hadComment bool } // Filter removes or preserves C# comments according to mode. @@ -86,34 +100,22 @@ type csharpLineFilter struct { // // Returns filtered source lines. func (CSharpCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string state := csharpState{} - for _, line := range lines { - filteredLine, hadComment := filterCSharpLine(line, mode, &state) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } - - return filtered -} -// filterCSharpLine removes or preserves recognized C# comments from one line. -func filterCSharpLine(line string, mode Mode, state *csharpState) (string, bool) { - filter := csharpLineFilter{ - line: line, - mode: mode, - state: state, - } + return filterLines(lines, func(line string) (string, bool) { + filter := csharpLineFilter{ + lineFilter: lineFilter{line: line, mode: mode}, + state: &state, + } - return filter.filterLine() + return filter.filterLine() + }) } // filterLine walks the current line until it reaches its end or a line comment. func (f *csharpLineFilter) filterLine() (string, bool) { for f.position < len(f.line) { - if f.consumeActiveBlock() { + if f.consumeActiveBlock(&f.state.block) { continue } if f.consumeStringInterpolation() { @@ -122,7 +124,7 @@ func (f *csharpLineFilter) filterLine() (string, bool) { if f.consumeStringText() { continue } - if f.consumeCharacterLiteral() { + if f.consumeQuotedSegment("'") { continue } if f.consumeStringStart() { @@ -143,31 +145,6 @@ func (f *csharpLineFilter) filterLine() (string, bool) { return f.result.String(), f.hadComment } -// consumeActiveBlock consumes text while the scanner is inside a block comment. -func (f *csharpLineFilter) consumeActiveBlock() bool { - if !f.state.blockActive { - return false - } - f.hadComment = true - end := strings.Index(f.line[f.position:], cStyleBlockCommentEnd) - if end < 0 { - if f.state.blockKeep { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return true - } - endPosition := f.position + end + len(cStyleBlockCommentEnd) - if f.state.blockKeep { - f.result.WriteString(f.line[f.position:endPosition]) - } - f.position = endPosition - f.state.blockActive = false - - return true -} - // consumeStringInterpolation filters code inside an active interpolation expression. func (f *csharpLineFilter) consumeStringInterpolation() bool { if !f.state.stringActive || f.state.interpolationDepth == 0 { @@ -181,7 +158,7 @@ func (f *csharpLineFilter) consumeStringInterpolation() bool { continue } - if f.consumeInterpolationCodeByte() { + if f.consumeInterpolationCodeByte(csharpInterpolation, &f.state.interpolationDepth) { return true } } @@ -191,7 +168,7 @@ func (f *csharpLineFilter) consumeStringInterpolation() bool { // consumeInterpolationSegment consumes multi-byte interpolation content. func (f *csharpLineFilter) consumeInterpolationSegment() bool { - if f.consumeActiveBlock() { + if f.consumeActiveBlock(&f.state.block) { return true } if f.consumeInterpolationFormat() { @@ -207,26 +184,6 @@ func (f *csharpLineFilter) consumeInterpolationSegment() bool { return false } -// consumeInterpolationCodeByte copies expression code and reports whether interpolation closed. -func (f *csharpLineFilter) consumeInterpolationCodeByte() bool { - switch f.line[f.position] { - case '{': - f.state.interpolationDepth++ - f.consumeCodeByte() - - return false - case '}': - f.state.interpolationDepth-- - f.consumeCodeByte() - - return f.state.interpolationDepth == 0 - default: - f.consumeCodeByte() - - return false - } -} - // consumeInterpolationFormat copies C# format text after a top-level interpolation colon. // This lightweight scanner does not track parentheses inside interpolation expressions, // so a ternary colon in parentheses is treated as format text. @@ -255,18 +212,19 @@ func (f *csharpLineFilter) consumeInterpolationFormat() bool { // consumeInterpolationString copies a line-local string literal inside interpolation code. // Nested multi-line verbatim strings intentionally resume as interpolation code on the next line. func (f *csharpLineFilter) consumeInterpolationString() bool { - token, verbatim, found := csharpStringTokenAt(f.line, f.position) + start, found := csharpStringStartAt(f.line, f.position) if !found { return false } - f.result.WriteString(token) - f.position += len(token) + f.consumeMarker(start.token) + if start.rawDelimiter != "" { + return f.consumeLineLocalRawString(start.rawDelimiter) + } for f.position < len(f.line) { switch { - case verbatim && strings.HasPrefix(f.line[f.position:], csharpEscapedQuote): - f.result.WriteString(csharpEscapedQuote) - f.position += len(csharpEscapedQuote) - case !verbatim && f.line[f.position] == '\\': + case start.verbatim && f.hasPrefix(csharpEscapedQuote): + f.consumeMarker(csharpEscapedQuote) + case !start.verbatim && f.line[f.position] == '\\': f.writeEscapedByte() case f.line[f.position] == '"': f.consumeCodeByte() @@ -280,6 +238,20 @@ func (f *csharpLineFilter) consumeInterpolationString() bool { return true } +// consumeLineLocalRawString copies raw string text until the closing delimiter or line end. +func (f *csharpLineFilter) consumeLineLocalRawString(delimiter string) bool { + for f.position < len(f.line) { + if f.hasPrefix(delimiter) { + f.consumeMarker(delimiter) + + return true + } + f.consumeCodeByte() + } + + return true +} + // consumeStringText copies active string text without scanning comment markers inside it. func (f *csharpLineFilter) consumeStringText() bool { if !f.state.stringActive || f.state.interpolationDepth > 0 { @@ -297,18 +269,21 @@ func (f *csharpLineFilter) consumeStringText() bool { // consumeStringTextSegment consumes special syntax inside active string text. func (f *csharpLineFilter) consumeStringTextSegment() bool { + if f.state.stringRawDelimiter != "" { + return f.consumeRawStringTextSegment() + } + + return f.consumeRegularStringTextSegment() +} + +// consumeRawStringTextSegment consumes special syntax inside active raw string text. +func (f *csharpLineFilter) consumeRawStringTextSegment() bool { switch { - case f.state.stringVerbatim && strings.HasPrefix(f.line[f.position:], csharpEscapedQuote): - f.result.WriteString(csharpEscapedQuote) - f.position += len(csharpEscapedQuote) - case !f.state.stringVerbatim && f.line[f.position] == '\\': - f.writeEscapedByte() - case f.line[f.position] == '"': - f.consumeCodeByte() + case f.hasPrefix(f.state.stringRawDelimiter): + f.consumeMarker(f.state.stringRawDelimiter) f.closeString() case f.startsEscapedInterpolationBrace(): - f.result.WriteString(f.line[f.position : f.position+2]) - f.position += 2 + f.consumeMarker(f.line[f.position : f.position+2]) case f.state.stringInterpolated && f.line[f.position] == '{': f.consumeCodeByte() f.state.interpolationDepth = 1 @@ -319,70 +294,101 @@ func (f *csharpLineFilter) consumeStringTextSegment() bool { return true } -// consumeCharacterLiteral copies a C# character literal. -func (f *csharpLineFilter) consumeCharacterLiteral() bool { - quoteEnd := quotedSegmentEnd(f.line, f.position, "'") - if quoteEnd <= f.position { +// consumeRegularStringTextSegment consumes special syntax inside regular string text. +func (f *csharpLineFilter) consumeRegularStringTextSegment() bool { + switch { + case f.state.stringVerbatim && f.hasPrefix(csharpEscapedQuote): + f.consumeMarker(csharpEscapedQuote) + case !f.state.stringVerbatim && f.line[f.position] == '\\': + f.writeEscapedByte() + case f.line[f.position] == '"': + f.consumeCodeByte() + f.closeString() + case f.startsEscapedInterpolationBrace(): + f.consumeMarker(f.line[f.position : f.position+2]) + case f.state.stringInterpolated && f.line[f.position] == '{': + f.consumeCodeByte() + f.state.interpolationDepth = 1 + default: return false } - f.result.WriteString(f.line[f.position:quoteEnd]) - f.position = quoteEnd return true } // consumeStringStart starts a C# string literal at the current position. func (f *csharpLineFilter) consumeStringStart() bool { - token, verbatim, found := csharpStringTokenAt(f.line, f.position) + start, found := csharpStringStartAt(f.line, f.position) if !found { return false } - interpolated := strings.HasPrefix(token, "$") || strings.HasPrefix(token, "@$") - f.startString(token, verbatim, interpolated) + f.consumeMarker(start.token) + f.state.stringActive = true + f.state.stringVerbatim = start.verbatim + f.state.stringInterpolated = start.interpolated + f.state.stringRawDelimiter = start.rawDelimiter return true } -// csharpStringTokenAt returns a C# string opener at position. -func csharpStringTokenAt(line string, position int) (string, bool, bool) { +// csharpStringStartAt returns a C# string opener at position. +func csharpStringStartAt(line string, position int) (csharpStringStart, bool) { switch { + case strings.HasPrefix(line[position:], csharpInterpolatedRawStringStart): + return csharpStringStart{ + token: csharpInterpolatedRawStringStart, + interpolated: true, + rawDelimiter: csharpRawStringDelimiter, + }, true case strings.HasPrefix(line[position:], csharpInterpolatedVerbatimStringStart): - return csharpInterpolatedVerbatimStringStart, true, true + return csharpStringStart{ + token: csharpInterpolatedVerbatimStringStart, + verbatim: true, + interpolated: true, + }, true case strings.HasPrefix(line[position:], csharpVerbatimInterpolatedStringStart): - return csharpVerbatimInterpolatedStringStart, true, true + return csharpStringStart{ + token: csharpVerbatimInterpolatedStringStart, + verbatim: true, + interpolated: true, + }, true case strings.HasPrefix(line[position:], csharpVerbatimStringStart): - return csharpVerbatimStringStart, true, true + return csharpStringStart{ + token: csharpVerbatimStringStart, + verbatim: true, + }, true case strings.HasPrefix(line[position:], csharpInterpolatedStringStart): - return csharpInterpolatedStringStart, false, true + return csharpStringStart{ + token: csharpInterpolatedStringStart, + interpolated: true, + }, true + case strings.HasPrefix(line[position:], csharpRawStringDelimiter): + return csharpStringStart{ + token: csharpRawStringDelimiter, + rawDelimiter: csharpRawStringDelimiter, + }, true case line[position] == '"': - return `"`, false, true + return csharpStringStart{token: `"`}, true default: - return "", false, false + return csharpStringStart{}, false } } -// startString records the active string literal and copies its opening token. -func (f *csharpLineFilter) startString(token string, verbatim bool, interpolated bool) { - f.result.WriteString(token) - f.position += len(token) - f.state.stringActive = true - f.state.stringVerbatim = verbatim - f.state.stringInterpolated = interpolated -} - // startsEscapedInterpolationBrace reports whether the position starts {{ or }} in string text. func (f *csharpLineFilter) startsEscapedInterpolationBrace() bool { if !f.state.stringInterpolated || f.position+1 >= len(f.line) { return false } - return strings.HasPrefix(f.line[f.position:], csharpEscapedOpenBrace) || - strings.HasPrefix(f.line[f.position:], csharpEscapedCloseBrace) + return f.hasPrefix(csharpEscapedOpenBrace) || f.hasPrefix(csharpEscapedCloseBrace) } // closeSingleLineStringAtLineEnd clears invalid single-line strings at end of line. func (f *csharpLineFilter) closeSingleLineStringAtLineEnd() { - if f.state.stringActive && !f.state.stringVerbatim && f.state.interpolationDepth == 0 { + if f.state.stringActive && + !f.state.stringVerbatim && + f.state.stringRawDelimiter == "" && + f.state.interpolationDepth == 0 { f.closeString() } } @@ -392,63 +398,17 @@ func (f *csharpLineFilter) closeString() { f.state.stringActive = false f.state.stringVerbatim = false f.state.stringInterpolated = false + f.state.stringRawDelimiter = "" f.state.interpolationDepth = 0 f.state.interpolationFormat = false } // consumeComment consumes a C# comment when one starts at the scanner position. func (f *csharpLineFilter) consumeComment() commentConsumeResult { - if strings.HasPrefix(f.line[f.position:], cStyleDocCommentStart) { - f.startBlockComment(f.mode == RetainDocumentation) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart) { - f.startBlockComment(f.mode == RetainBlock || f.mode == RetainRegular) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], "///") { - f.hadComment = true - if f.mode == RetainDocumentation { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return commentConsumeResult{consumed: true, stopLine: true} - } - if strings.HasPrefix(f.line[f.position:], "//") { - f.hadComment = true - if f.mode == RetainInline || f.mode == RetainRegular { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return commentConsumeResult{consumed: true, stopLine: true} - } - - return commentConsumeResult{} + return f.consumeCStyleComment(csharpDocLineComment, f.startBlockComment) } // startBlockComment records the active block comment and whether to keep it. func (f *csharpLineFilter) startBlockComment(keep bool) { - f.hadComment = true - f.state.blockActive = true - f.state.blockKeep = keep -} - -// writeEscapedByte copies an escaped byte pair from a regular string. -func (f *csharpLineFilter) writeEscapedByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ - if f.position < len(f.line) { - f.result.WriteByte(f.line[f.position]) - f.position++ - } -} - -// consumeCodeByte copies one source byte. -func (f *csharpLineFilter) consumeCodeByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ + f.lineFilter.startBlockComment(&f.state.block, keep) } diff --git a/embedding/commentfilter/csharp_filter_test.go b/embedding/commentfilter/csharp_filter_test.go index e77edc8..eecbf48 100644 --- a/embedding/commentfilter/csharp_filter_test.go +++ b/embedding/commentfilter/csharp_filter_test.go @@ -121,4 +121,40 @@ var _ = Describe("C#", func() { assertFiltered("Api.cs", RetainNone, lines, expected) }) + + It("should strip comments without treating raw string text as comments", func() { + lines := []string{ + `var raw = """`, + `Keep // text`, + `Keep /* text */`, + `"""; // trailing comment`, + `var interpolated = $"""`, + `Keep // text {Format(value /* real comment */)}`, + `"""; // trailing comment`, + } + + expected := []string{ + `var raw = """`, + `Keep // text`, + `Keep /* text */`, + `"""; `, + `var interpolated = $"""`, + `Keep // text {Format(value )}`, + `"""; `, + } + + assertFiltered("Api.cs", RetainNone, lines, expected) + }) + + It("should not close block comments on an overlapping end marker", func() { + lines := []string{ + "var before = 1; /*/ still comment */ var after = 2;", + } + + expected := []string{ + "var before = 1; var after = 2;", + } + + assertFiltered("Api.cs", RetainNone, lines, expected) + }) }) diff --git a/embedding/commentfilter/filter.go b/embedding/commentfilter/filter.go index b29eca4..166addf 100644 --- a/embedding/commentfilter/filter.go +++ b/embedding/commentfilter/filter.go @@ -23,6 +23,7 @@ import ( "fmt" "log/slog" "path/filepath" + "slices" "strings" ) @@ -50,6 +51,23 @@ type CommentFilter interface { Filter(lines []string, mode Mode) []string } +// filterLines applies a line filter and drops lines made empty by comment removal. +func filterLines( + lines []string, + filterLine func(line string) (filteredLine string, hadComment bool), +) []string { + var filtered []string + for _, line := range lines { + filteredLine, hadComment := filterLine(line) + if hadComment && strings.TrimSpace(filteredLine) == "" { + continue + } + filtered = append(filtered, filteredLine) + } + + return filtered +} + // Filter returns source lines with comments stripped according to the requested mode. // // Parameters: @@ -161,7 +179,7 @@ func warnUnsupportedCommentsMode( embeddingLine int, supportedModes []Mode, ) bool { - if containsMode(supportedModes, mode) { + if slices.Contains(supportedModes, mode) { return false } var wrappedModes []string @@ -182,14 +200,3 @@ func warnUnsupportedCommentsMode( return true } - -// containsMode reports whether the list includes the given mode. -func containsMode(modes []Mode, mode Mode) bool { - for _, supportedMode := range modes { - if supportedMode == mode { - return true - } - } - - return false -} diff --git a/embedding/commentfilter/javascript_filter.go b/embedding/commentfilter/javascript_filter.go index 57332d2..2ce95e7 100644 --- a/embedding/commentfilter/javascript_filter.go +++ b/embedding/commentfilter/javascript_filter.go @@ -20,18 +20,29 @@ package commentfilter import "strings" -const jsTemplateInterpolationStart = "${" +const jsTemplateDelimiter = "`" + +// jsInterpolation describes the JavaScript `${...}` template interpolation form. +var jsInterpolation = interpolationForm{ + start: "${", + openBrace: '{', + closeBrace: '}', +} + +// jsTemplateLiteral describes the JavaScript template literal form. +var jsTemplateLiteral = interpolatedLiteral{ + delimiter: jsTemplateDelimiter, + escapes: true, + interpolation: jsInterpolation, +} // JavaScriptCommentFilter filters JavaScript and TypeScript comments while preserving literal text. type JavaScriptCommentFilter struct{} // javascriptState tracks JavaScript lexical state that can span source lines. type javascriptState struct { - // blockActive reports whether scanning is inside a block comment. - blockActive bool - - // blockKeep reports whether the active block comment should be retained. - blockKeep bool + // block tracks a block comment across source lines. + block blockCommentState // template reports whether scanning is inside template literal text. template bool @@ -48,41 +59,11 @@ type javascriptState struct { // javascriptLineFilter filters one JavaScript or TypeScript source line. type javascriptLineFilter struct { - // line is the source line being filtered. - line string - - // mode selects which comments to retain. - mode Mode + // lineFilter provides shared line scanning state and helpers. + lineFilter // state tracks JavaScript constructs across lines. state *javascriptState - - // result accumulates the filtered source line. - result strings.Builder - - // position is the current byte index in line. - position int - - // hadComment reports whether the line contained a recognized comment. - hadComment bool -} - -// commentConsumeResult describes a consumed JavaScript comment. -type commentConsumeResult struct { - // consumed reports whether a recognized comment marker was consumed. - consumed bool - - // stopLine reports whether the consumed comment reaches the end of the source line. - stopLine bool -} - -// interpolationCodeResult describes the effect of one consumed interpolation byte. -type interpolationCodeResult struct { - // depth is the brace depth after consuming the byte at the scanner position. - depth int - - // closed reports whether the consumed byte closed the current interpolation expression. - closed bool } // Filter removes or preserves JavaScript and TypeScript comments according to mode. @@ -93,34 +74,22 @@ type interpolationCodeResult struct { // // Returns filtered source lines. func (JavaScriptCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string state := javascriptState{} - for _, line := range lines { - filteredLine, hadComment := filterJavaScriptLine(line, mode, &state) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } - return filtered -} - -// filterJavaScriptLine removes or preserves recognized JavaScript comments from one line. -func filterJavaScriptLine(line string, mode Mode, state *javascriptState) (string, bool) { - filter := javascriptLineFilter{ - line: line, - mode: mode, - state: state, - } + return filterLines(lines, func(line string) (string, bool) { + filter := javascriptLineFilter{ + lineFilter: lineFilter{line: line, mode: mode}, + state: &state, + } - return filter.filterLine() + return filter.filterLine() + }) } // filterLine walks the current line until it reaches its end or a line comment. func (f *javascriptLineFilter) filterLine() (string, bool) { for f.position < len(f.line) { - if f.consumeActiveBlock() { + if f.consumeActiveBlock(&f.state.block) { continue } if f.consumeTemplateInterpolation() { @@ -148,31 +117,6 @@ func (f *javascriptLineFilter) filterLine() (string, bool) { return f.result.String(), f.hadComment } -// consumeActiveBlock consumes text while the scanner is inside a block comment. -func (f *javascriptLineFilter) consumeActiveBlock() bool { - if !f.state.blockActive { - return false - } - f.hadComment = true - end := strings.Index(f.line[f.position:], cStyleBlockCommentEnd) - if end < 0 { - if f.state.blockKeep { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return true - } - endPosition := f.position + end + len(cStyleBlockCommentEnd) - if f.state.blockKeep { - f.result.WriteString(f.line[f.position:endPosition]) - } - f.position = endPosition - f.state.blockActive = false - - return true -} - // consumeTemplateInterpolation resumes JavaScript expression scanning inside `${...}`. func (f *javascriptLineFilter) consumeTemplateInterpolation() bool { if f.state.templateInterpolationDepth == 0 { @@ -191,54 +135,17 @@ func (f *javascriptLineFilter) consumeTemplateInterpolation() bool { // consumeTemplateText copies template text and filters comments inside `${...}` code. func (f *javascriptLineFilter) consumeTemplateText() bool { - if !f.state.template && f.line[f.position] != '`' { - return false - } - if !f.state.template { - f.state.template = true - f.consumeCodeByte() - } - for f.position < len(f.line) { - switch { - case f.line[f.position] == '\\': - f.writeEscapedByte() - case f.line[f.position] == '`': - f.consumeCodeByte() - f.state.template = false - - return true - case strings.HasPrefix(f.line[f.position:], jsTemplateInterpolationStart): - f.result.WriteString(jsTemplateInterpolationStart) - f.position += len(jsTemplateInterpolationStart) - f.state.template = false - f.state.templateInterpolationDepth = 1 - f.consumeTemplateInterpolation() - if f.state.templateInterpolationDepth > 0 { - return true - } - default: - f.consumeCodeByte() - } - } - - return true + return f.consumeInterpolatedText( + jsTemplateLiteral, + &f.state.template, + &f.state.templateInterpolationDepth, + f.consumeTemplateInterpolation, + ) } // consumeString copies a quoted string without scanning comment markers inside it. func (f *javascriptLineFilter) consumeString() bool { - if f.position >= len(f.line) { - return false - } - switch f.line[f.position] { - case '"', '\'': - quoteEnd := quotedSegmentEnd(f.line, f.position, "\"'") - f.result.WriteString(f.line[f.position:quoteEnd]) - f.position = quoteEnd - - return true - default: - return false - } + return f.consumeQuotedSegment("\"'") } // consumeRegexLiteral copies a regular-expression literal without treating its content as comments. @@ -281,8 +188,7 @@ func (f *javascriptLineFilter) regexStartsHere() bool { if f.line[f.position] != '/' { return false } - if strings.HasPrefix(f.line[f.position:], "//") || - strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart) { + if f.hasPrefix("//") || f.hasPrefix(cStyleBlockCommentStart) { return false } previous := previousSignificantToken(f.line[:f.position]) @@ -319,7 +225,7 @@ func (f *javascriptLineFilter) consumeRegexFlags() { // depth - current brace depth of the interpolation expression; updated in place. func (f *javascriptLineFilter) consumeInterpolationDepth(depth *int) { for f.position < len(f.line) { - if f.consumeActiveBlock() { + if f.consumeActiveBlock(&f.state.block) { continue } if f.consumeNestedTemplateLiteral() { @@ -338,11 +244,7 @@ func (f *javascriptLineFilter) consumeInterpolationDepth(depth *int) { continue } - code := f.consumeInterpolationCode(*depth) - *depth = code.depth - if code.closed { - *depth = 0 - + if f.consumeInterpolationCodeByte(jsInterpolation, depth) { return } } @@ -350,33 +252,12 @@ func (f *javascriptLineFilter) consumeInterpolationDepth(depth *int) { // consumeNestedTemplateLiteral copies a template literal found inside interpolation code. func (f *javascriptLineFilter) consumeNestedTemplateLiteral() bool { - if !f.startOrResumeNestedTemplateLiteral() { - return false - } - for f.position < len(f.line) { - switch { - case f.line[f.position] == '\\': - f.writeEscapedByte() - case f.line[f.position] == '`': - f.consumeCodeByte() - f.state.nestedTemplate = false - - return true - case strings.HasPrefix(f.line[f.position:], jsTemplateInterpolationStart): - f.result.WriteString(jsTemplateInterpolationStart) - f.position += len(jsTemplateInterpolationStart) - f.state.nestedTemplate = false - f.state.nestedTemplateInterpolationDepth = 1 - f.consumeNestedTemplateInterpolation() - if f.state.nestedTemplateInterpolationDepth > 0 { - return true - } - default: - f.consumeCodeByte() - } - } - - return true + return f.consumeInterpolatedText( + jsTemplateLiteral, + &f.state.nestedTemplate, + &f.state.nestedTemplateInterpolationDepth, + f.consumeNestedTemplateInterpolation, + ) } // consumeNestedTemplateInterpolation resumes code inside nested template `${...}`. @@ -392,93 +273,14 @@ func (f *javascriptLineFilter) consumeNestedTemplateInterpolation() bool { return true } -// startOrResumeNestedTemplateLiteral enters or resumes nested template scanning. -func (f *javascriptLineFilter) startOrResumeNestedTemplateLiteral() bool { - if f.state.nestedTemplate { - return true - } - if f.position >= len(f.line) || f.line[f.position] != '`' { - return false - } - f.state.nestedTemplate = true - f.consumeCodeByte() - - return true -} - -// consumeInterpolationCode copies expression code and updates interpolation brace depth. -// -// Parameters: -// depth - current brace depth before consuming the byte at the scanner position. -// -// Returns interpolation code result. -func (f *javascriptLineFilter) consumeInterpolationCode(depth int) interpolationCodeResult { - switch f.line[f.position] { - case '{': - depth++ - f.consumeCodeByte() - - return interpolationCodeResult{depth: depth} - case '}': - depth-- - f.consumeCodeByte() - - return interpolationCodeResult{depth: depth, closed: depth == 0} - default: - f.consumeCodeByte() - - return interpolationCodeResult{depth: depth} - } -} - // consumeComment consumes a JavaScript comment when one starts at the scanner position. -// -// Returns comment consume result. func (f *javascriptLineFilter) consumeComment() commentConsumeResult { - if strings.HasPrefix(f.line[f.position:], cStyleDocCommentStart) { - f.startBlockComment(f.mode == RetainDocumentation) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart) { - f.startBlockComment(f.mode == RetainBlock || f.mode == RetainRegular) - - return commentConsumeResult{consumed: true} - } - if strings.HasPrefix(f.line[f.position:], "//") { - f.hadComment = true - if f.mode == RetainInline || f.mode == RetainRegular { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return commentConsumeResult{consumed: true, stopLine: true} - } - - return commentConsumeResult{} + return f.consumeCStyleComment("", f.startBlockComment) } -// startBlockComment records the active block comment markers and whether to keep them. +// startBlockComment records the active block comment and whether to keep it. func (f *javascriptLineFilter) startBlockComment(keep bool) { - f.hadComment = true - f.state.blockActive = true - f.state.blockKeep = keep -} - -// writeEscapedByte copies an escaped byte pair from a literal. -func (f *javascriptLineFilter) writeEscapedByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ - if f.position < len(f.line) { - f.result.WriteByte(f.line[f.position]) - f.position++ - } -} - -// consumeCodeByte copies one source byte. -func (f *javascriptLineFilter) consumeCodeByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ + f.lineFilter.startBlockComment(&f.state.block, keep) } // previousSignificantToken returns the last non-space token in text. diff --git a/embedding/commentfilter/javascript_filter_test.go b/embedding/commentfilter/javascript_filter_test.go index 2789867..1d11420 100644 --- a/embedding/commentfilter/javascript_filter_test.go +++ b/embedding/commentfilter/javascript_filter_test.go @@ -173,4 +173,16 @@ var _ = Describe("JavaScript and TypeScript", func() { assertFiltered("sample.ts", tc.mode, lines, tc.expected) } }) + + It("should not close block comments on an overlapping end marker", func() { + lines := []string{ + "const before = 1; /*/ still comment */ const after = 2;", + } + + expected := []string{ + "const before = 1; const after = 2;", + } + + assertFiltered("sample.ts", RetainNone, lines, expected) + }) }) diff --git a/embedding/commentfilter/kotlin_filter.go b/embedding/commentfilter/kotlin_filter.go index 4990306..1366973 100644 --- a/embedding/commentfilter/kotlin_filter.go +++ b/embedding/commentfilter/kotlin_filter.go @@ -18,10 +18,22 @@ package commentfilter -import "strings" - const kotlinRawStringDelimiter = "\"\"\"" +// kotlinInterpolation describes the Kotlin `${...}` string interpolation form. +var kotlinInterpolation = interpolationForm{ + start: "${", + openBrace: '{', + closeBrace: '}', +} + +// kotlinRawStringLiteral describes the Kotlin raw triple-quoted string form. +// Raw strings have no backslash escapes. +var kotlinRawStringLiteral = interpolatedLiteral{ + delimiter: kotlinRawStringDelimiter, + interpolation: kotlinInterpolation, +} + // KotlinCommentFilter filters Kotlin comments while preserving Kotlin string forms. type KotlinCommentFilter struct{} @@ -42,23 +54,11 @@ type kotlinState struct { // kotlinLineFilter filters one Kotlin source line. type kotlinLineFilter struct { - // line is the source line being filtered. - line string - - // mode selects which comments to retain. - mode Mode + // lineFilter provides shared line scanning state and helpers. + lineFilter // state tracks Kotlin constructs across lines. state *kotlinState - - // result accumulates the filtered source line. - result strings.Builder - - // position is the current byte index in line. - position int - - // hadComment reports whether the line contained a recognized comment. - hadComment bool } // Filter removes or preserves Kotlin comments according to mode. @@ -69,28 +69,16 @@ type kotlinLineFilter struct { // // Returns filtered source lines. func (KotlinCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string state := kotlinState{} - for _, line := range lines { - filteredLine, hadComment := filterKotlinLine(line, mode, &state) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } - return filtered -} - -// filterKotlinLine removes or preserves recognized Kotlin comments from one line. -func filterKotlinLine(line string, mode Mode, state *kotlinState) (string, bool) { - filter := kotlinLineFilter{ - line: line, - mode: mode, - state: state, - } + return filterLines(lines, func(line string) (string, bool) { + filter := kotlinLineFilter{ + lineFilter: lineFilter{line: line, mode: mode}, + state: &state, + } - return filter.filterLine() + return filter.filterLine() + }) } // filterLine walks the current line until it reaches its end or a line comment. @@ -108,8 +96,8 @@ func (f *kotlinLineFilter) filterLine() (string, bool) { if f.consumeString() { continue } - if consumed, stop := f.consumeComment(); consumed { - if stop { + if comment := f.consumeComment(); comment.consumed { + if comment.stopLine { break } @@ -129,11 +117,11 @@ func (f *kotlinLineFilter) consumeActiveBlock() bool { f.hadComment = true for f.position < len(f.line) { switch { - case strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart): + case f.hasPrefix(cStyleBlockCommentStart): f.writeBlockText(cStyleBlockCommentStart) f.state.blockDepth++ f.position += len(cStyleBlockCommentStart) - case strings.HasPrefix(f.line[f.position:], cStyleBlockCommentEnd): + case f.hasPrefix(cStyleBlockCommentEnd): f.writeBlockText(cStyleBlockCommentEnd) f.state.blockDepth-- f.position += len(cStyleBlockCommentEnd) @@ -155,37 +143,12 @@ func (f *kotlinLineFilter) consumeActiveBlock() bool { // // It treats the first three quotes in a run of four or more quotes as the raw-string delimiter. func (f *kotlinLineFilter) consumeRawString() bool { - if !f.state.rawString && !strings.HasPrefix(f.line[f.position:], kotlinRawStringDelimiter) { - return false - } - if !f.state.rawString { - f.state.rawString = true - f.result.WriteString(kotlinRawStringDelimiter) - f.position += len(kotlinRawStringDelimiter) - } - for f.position < len(f.line) { - switch { - case strings.HasPrefix(f.line[f.position:], kotlinRawStringDelimiter): - f.result.WriteString(kotlinRawStringDelimiter) - f.position += len(kotlinRawStringDelimiter) - f.state.rawString = false - - return true - case strings.HasPrefix(f.line[f.position:], "${"): - f.result.WriteString("${") - f.position += len("${") - f.state.rawString = false - f.state.rawInterpolationDepth = 1 - f.consumeRawInterpolation() - if f.state.rawInterpolationDepth > 0 { - return true - } - default: - f.consumeCodeByte() - } - } - - return true + return f.consumeInterpolatedText( + kotlinRawStringLiteral, + &f.state.rawString, + &f.state.rawInterpolationDepth, + f.consumeRawInterpolation, + ) } // consumeRawInterpolation resumes Kotlin expression scanning inside a raw-string interpolation. @@ -212,11 +175,7 @@ func (f *kotlinLineFilter) consumeString() bool { return true case '\'': - quoteEnd := quotedSegmentEnd(f.line, f.position, "'") - f.result.WriteString(f.line[f.position:quoteEnd]) - f.position = quoteEnd - - return true + return f.consumeQuotedSegment("'") default: return false } @@ -224,20 +183,17 @@ func (f *kotlinLineFilter) consumeString() bool { // consumeQuotedString copies a Kotlin quoted string and filters comments inside `${...}`. func (f *kotlinLineFilter) consumeQuotedString() { - f.result.WriteByte(f.line[f.position]) - f.position++ + f.consumeCodeByte() for f.position < len(f.line) { switch { case f.line[f.position] == '\\': f.writeEscapedByte() case f.line[f.position] == '"': - f.result.WriteByte(f.line[f.position]) - f.position++ + f.consumeCodeByte() return - case strings.HasPrefix(f.line[f.position:], "${"): - f.result.WriteString("${") - f.position += len("${") + case f.hasPrefix(kotlinInterpolation.start): + f.consumeMarker(kotlinInterpolation.start) f.consumeInterpolation() default: f.consumeCodeByte() @@ -264,66 +220,22 @@ func (f *kotlinLineFilter) consumeInterpolationDepth(depth *int) { if f.consumeString() { continue } - if consumed, stop := f.consumeComment(); consumed { - if stop { + if comment := f.consumeComment(); comment.consumed { + if comment.stopLine { return } continue } - var done bool - *depth, done = f.consumeInterpolationCode(*depth) - if done { - *depth = 0 - + if f.consumeInterpolationCodeByte(kotlinInterpolation, depth) { return } } } -// consumeInterpolationCode copies expression code and updates interpolation brace depth. -func (f *kotlinLineFilter) consumeInterpolationCode(depth int) (int, bool) { - switch f.line[f.position] { - case '{': - depth++ - f.consumeCodeByte() - - return depth, false - case '}': - depth-- - f.consumeCodeByte() - - return depth, depth == 0 - default: - f.consumeCodeByte() - - return depth, false - } -} - -// consumeComment consumes a Kotlin comment and reports whether it ended the line. -func (f *kotlinLineFilter) consumeComment() (bool, bool) { - if strings.HasPrefix(f.line[f.position:], cStyleDocCommentStart) { - f.startBlockComment(f.mode == RetainDocumentation) - - return true, false - } - if strings.HasPrefix(f.line[f.position:], cStyleBlockCommentStart) { - f.startBlockComment(f.mode == RetainBlock || f.mode == RetainRegular) - - return true, false - } - if strings.HasPrefix(f.line[f.position:], "//") { - f.hadComment = true - if f.mode == RetainInline || f.mode == RetainRegular { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) - - return true, true - } - - return false, false +// consumeComment consumes a Kotlin comment when one starts at the scanner position. +func (f *kotlinLineFilter) consumeComment() commentConsumeResult { + return f.consumeCStyleComment("", f.startBlockComment) } // startBlockComment starts a Kotlin block comment with nesting depth one. @@ -343,19 +255,3 @@ func (f *kotlinLineFilter) writeBlockText(text string) { f.result.WriteString(text) } } - -// writeEscapedByte copies an escaped byte pair from a quoted string. -func (f *kotlinLineFilter) writeEscapedByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ - if f.position < len(f.line) { - f.result.WriteByte(f.line[f.position]) - f.position++ - } -} - -// consumeCodeByte copies one source byte. -func (f *kotlinLineFilter) consumeCodeByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ -} diff --git a/embedding/commentfilter/line_filter.go b/embedding/commentfilter/line_filter.go new file mode 100644 index 0000000..8147b72 --- /dev/null +++ b/embedding/commentfilter/line_filter.go @@ -0,0 +1,311 @@ +// Copyright 2026, TeamDev. All rights reserved. +// +// Redistribution and use in source and/or binary forms, with or without +// modification, must retain the above copyright notice and the following +// disclaimer. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package commentfilter + +import "strings" + +// interpolationForm describes how a language opens and closes interpolation expressions. +// +// start and openBrace play different roles: start is the marker that opens an +// interpolation in literal text, while openBrace is the plain brace that nests +// inside expression code. In `${items.map(i => { return i })}` the interpolation +// opens with `${`, but the lambda `{` — which no longer follows a `$` — must +// still deepen the nesting so that its matching closeBrace does not end the +// interpolation early. +type interpolationForm struct { + // start opens an interpolation expression in literal text, e.g. `${`. + start string + + // openBrace nests the expression code one level deeper. + openBrace byte + + // closeBrace closes one nesting level. + // + // The expression ends at the closeBrace that returns the nesting depth to zero. + closeBrace byte +} + +// interpolatedLiteral describes a string literal form that embeds interpolation expressions. +type interpolatedLiteral struct { + // delimiter opens and closes the literal text. + delimiter string + + // escapes reports whether backslashes escape literal bytes. + escapes bool + + // interpolation opens and closes interpolation expressions inside the literal. + interpolation interpolationForm +} + +// lineFilter carries the scanning state shared by the per-language line filters. +type lineFilter struct { + // line is the source line being filtered. + line string + + // mode selects which comments to retain. + mode Mode + + // result accumulates the filtered source line. + result strings.Builder + + // position is the current byte index in line. + position int + + // hadComment reports whether the line contained a recognized comment. + hadComment bool +} + +// commentConsumeResult describes a consumed source comment. +type commentConsumeResult struct { + // consumed reports whether a recognized comment marker was consumed. + consumed bool + + // stopLine reports whether the consumed comment reaches the end of the source line. + stopLine bool +} + +// blockCommentState tracks a non-nested block comment across source lines. +type blockCommentState struct { + // active reports whether scanning is inside a block comment. + active bool + + // keep reports whether the active block comment should be retained. + keep bool +} + +// hasPrefix reports whether text starts at the scanner position. +func (f *lineFilter) hasPrefix(text string) bool { + return strings.HasPrefix(f.line[f.position:], text) +} + +// consumeCodeByte copies one source byte. +func (f *lineFilter) consumeCodeByte() { + f.result.WriteByte(f.line[f.position]) + f.position++ +} + +// consumeMarker copies marker text and advances past it. +func (f *lineFilter) consumeMarker(marker string) { + f.result.WriteString(marker) + f.position += len(marker) +} + +// writeEscapedByte copies an escaped byte pair from a literal. +func (f *lineFilter) writeEscapedByte() { + f.result.WriteByte(f.line[f.position]) + f.position++ + if f.position < len(f.line) { + f.result.WriteByte(f.line[f.position]) + f.position++ + } +} + +// consumeQuotedSegment copies a quoted literal when one starts at the scanner position. +func (f *lineFilter) consumeQuotedSegment(quoteChars string) bool { + quoteEnd := quotedSegmentEnd(f.line, f.position, quoteChars) + if quoteEnd <= f.position { + return false + } + f.result.WriteString(f.line[f.position:quoteEnd]) + f.position = quoteEnd + + return true +} + +// quotedSegmentEnd returns the end offset of a quoted string starting at position. +func quotedSegmentEnd(line string, position int, quoteChars string) int { + if position >= len(line) || !strings.ContainsRune(quoteChars, rune(line[position])) { + return position + } + quote := line[position] + cursor := position + 1 + for cursor < len(line) { + if line[cursor] == '\\' { + cursor += 2 + + continue + } + if line[cursor] == quote { + return cursor + 1 + } + cursor++ + } + + return len(line) +} + +// consumeLineComment consumes the rest of the line as a line comment. +func (f *lineFilter) consumeLineComment(keep bool) { + f.hadComment = true + if keep { + f.result.WriteString(f.line[f.position:]) + } + f.position = len(f.line) +} + +// consumeCStyleComment consumes a C-style comment when one starts at the scanner position. +// +// Parameters: +// docLineMarker - optional documentation line-comment marker such as `///`; empty when absent. +// startBlock - language hook that records an opened block comment. +// +// Returns comment consume result. +func (f *lineFilter) consumeCStyleComment( + docLineMarker string, + startBlock func(keep bool), +) commentConsumeResult { + if f.hasPrefix(cStyleDocCommentStart) { + startBlock(f.mode == RetainDocumentation) + + return commentConsumeResult{consumed: true} + } + if f.hasPrefix(cStyleBlockCommentStart) { + startBlock(f.mode == RetainBlock || f.mode == RetainRegular) + + return commentConsumeResult{consumed: true} + } + if docLineMarker != "" && f.hasPrefix(docLineMarker) { + f.consumeLineComment(f.mode == RetainDocumentation) + + return commentConsumeResult{consumed: true, stopLine: true} + } + if f.hasPrefix("//") { + f.consumeLineComment(f.mode == RetainInline || f.mode == RetainRegular) + + return commentConsumeResult{consumed: true, stopLine: true} + } + + return commentConsumeResult{} +} + +// startBlockComment records and consumes an opened non-nested C-style block comment. +func (f *lineFilter) startBlockComment(state *blockCommentState, keep bool) { + f.hadComment = true + state.active = true + state.keep = keep + if keep { + f.result.WriteString(cStyleBlockCommentStart) + } + f.position += len(cStyleBlockCommentStart) +} + +// consumeActiveBlock consumes text while the scanner is inside a non-nested block comment. +func (f *lineFilter) consumeActiveBlock(state *blockCommentState) bool { + if !state.active { + return false + } + f.hadComment = true + end := strings.Index(f.line[f.position:], cStyleBlockCommentEnd) + if end < 0 { + if state.keep { + f.result.WriteString(f.line[f.position:]) + } + f.position = len(f.line) + + return true + } + endPosition := f.position + end + len(cStyleBlockCommentEnd) + if state.keep { + f.result.WriteString(f.line[f.position:endPosition]) + } + f.position = endPosition + state.active = false + + return true +} + +// consumeInterpolationCodeByte copies one expression byte and updates interpolation brace depth. +// +// Returns true when the consumed byte closed the interpolation expression. +func (f *lineFilter) consumeInterpolationCodeByte(form interpolationForm, depth *int) bool { + char := f.line[f.position] + f.consumeCodeByte() + switch char { + case form.openBrace: + *depth++ + case form.closeBrace: + *depth-- + + return *depth == 0 + } + + return false +} + +// consumeInterpolatedText copies interpolated literal text and enters interpolation code. +// +// Parameters: +// literal - describes the literal delimiter, escape rule, and interpolation opener. +// active - tracks whether the scanner is inside the literal text across lines. +// depth - tracks the interpolation brace depth across lines. +// resumeInterpolation - language hook that scans interpolation code until depth closes. +// +// Returns true when literal text was consumed. +func (f *lineFilter) consumeInterpolatedText( + literal interpolatedLiteral, + active *bool, + depth *int, + resumeInterpolation func() bool, +) bool { + if !*active && !f.hasPrefix(literal.delimiter) { + return false + } + if !*active { + *active = true + f.consumeMarker(literal.delimiter) + } + for f.position < len(f.line) { + if f.consumeInterpolatedTextSegment(literal, active, depth, resumeInterpolation) { + return true + } + } + + return true +} + +// consumeInterpolatedTextSegment consumes one piece of interpolated literal text. +// +// Returns true when the literal closed or its interpolation stayed open at the line end. +func (f *lineFilter) consumeInterpolatedTextSegment( + literal interpolatedLiteral, + active *bool, + depth *int, + resumeInterpolation func() bool, +) bool { + switch { + case literal.escapes && f.line[f.position] == '\\': + f.writeEscapedByte() + case f.hasPrefix(literal.delimiter): + f.consumeMarker(literal.delimiter) + *active = false + + return true + case f.hasPrefix(literal.interpolation.start): + f.consumeMarker(literal.interpolation.start) + *active = false + *depth = 1 + resumeInterpolation() + + return *depth > 0 + default: + f.consumeCodeByte() + } + + return false +} diff --git a/embedding/commentfilter/marker_comment_filter.go b/embedding/commentfilter/marker_comment_filter.go index d245331..78fa9c1 100644 --- a/embedding/commentfilter/marker_comment_filter.go +++ b/embedding/commentfilter/marker_comment_filter.go @@ -20,97 +20,37 @@ package commentfilter import "strings" -// BlockMarker describes a block comment marker pair. -type BlockMarker struct { - // Start is the block comment opening marker. - Start string +// activeSegment describes a multi-line construct currently being scanned. +type activeSegment struct { + // end closes the active construct. + end string - // End is the block comment closing marker. - End string -} - -// DocumentationMarker describes API documentation comment markers. -type DocumentationMarker struct { - // Inline contains documentation line-comment markers. - Inline []string - - // Block contains documentation block-comment marker pairs. - Block []BlockMarker -} - -// TextBlockMarker describes a multi-line text literal delimiter. -type TextBlockMarker struct { - // Delimiter opens and closes the text literal. - Delimiter string - - // Escapes reports whether backslashes escape delimiter bytes. - Escapes bool -} - -// CommentMarker describes lexical comment markers and string delimiters for a language family. -type CommentMarker struct { - // Inline contains line-comment markers. - Inline []string - - // Block contains block-comment marker pairs. - Block []BlockMarker - - // Documentation contains API documentation comment markers. - Documentation DocumentationMarker - - // TextBlocks contains markers that open and close multi-line text literals. - TextBlocks []TextBlockMarker + // keep reports whether the active construct is copied to output. + keep bool - // QuoteChars contains characters that open and close quoted strings. - QuoteChars string -} + // escapes reports whether backslashes escape bytes while searching for end. + escapes bool -// MarkerCommentFilter removes comments using lexical markers declared in CommentMarker. -type MarkerCommentFilter struct { - // Syntax contains the comment markers and string delimiters to recognize. - Syntax CommentMarker + // comment reports whether the active construct is a source comment. + comment bool } -// blockState tracks active multi-line lexical constructs across source lines. -type blockState struct { - // active reports whether scanning is inside a block comment. - active bool - - // block contains the active block comment markers. - block BlockMarker - - // keep reports whether the active comment should be retained. - keep bool - - // textBlockActive reports whether scanning is inside a text block. - textBlockActive bool - - // textBlock contains the active text block marker. - textBlock TextBlockMarker +// markerState tracks active multi-line lexical constructs across source lines. +type markerState struct { + // segment contains the active construct configuration, if one is open. + segment *activeSegment } // markerLineFilter tracks lexical comment filtering state for one source line. type markerLineFilter struct { + // lineFilter provides shared line scanning state and helpers. + lineFilter + // filter contains the language syntax configuration. filter MarkerCommentFilter - // line is the source line being filtered. - line string - - // mode selects which comments to retain. - mode Mode - // state tracks multi-line lexical constructs across lines. - state *blockState - - // result accumulates the filtered source line. - result strings.Builder - - // position is the current byte index in line. - position int - - // hadComment reports whether the line contained a recognized comment. - hadComment bool + state *markerState } // Filter removes or preserves recognized comments across all lines. @@ -121,52 +61,33 @@ type markerLineFilter struct { // // Returns filtered source lines. func (f MarkerCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string - state := blockState{} - for _, line := range lines { - filteredLine, hadComment := f.filterLine(line, mode, &state) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } + state := markerState{} - return filtered -} - -// filterLine removes or preserves recognized comments from a single source line. -func (f MarkerCommentFilter) filterLine( - line string, - mode Mode, - state *blockState, -) (string, bool) { - filter := markerLineFilter{ - filter: f, - line: line, - mode: mode, - state: state, - } + return filterLines(lines, func(line string) (string, bool) { + filter := markerLineFilter{ + lineFilter: lineFilter{line: line, mode: mode}, + filter: f, + state: &state, + } - return filter.filterLine() + return filter.filterLine() + }) } // filterLine walks the current line until it reaches its end or a line comment. func (f *markerLineFilter) filterLine() (string, bool) { for f.position < len(f.line) { - if f.consumeActiveBlock() { - continue - } - if f.consumeActiveTextBlock() { + if f.consumeActiveSegment() { continue } if f.consumeTextBlockStart() { continue } - if f.consumeQuotedSegment() { + if f.consumeQuotedSegment(f.filter.Syntax.QuoteChars) { continue } - if consumed, stop := f.consumeComment(); consumed { - if stop { + if comment := f.consumeComment(); comment.consumed { + if comment.stopLine { break } @@ -178,61 +99,43 @@ func (f *markerLineFilter) filterLine() (string, bool) { return f.result.String(), f.hadComment } -// consumeActiveBlock consumes text while the scanner is inside a block comment. -func (f *markerLineFilter) consumeActiveBlock() bool { - if !f.state.active { +// consumeActiveSegment consumes text while the scanner is inside a multi-line construct. +func (f *markerLineFilter) consumeActiveSegment() bool { + segment := f.state.segment + if segment == nil { return false } - f.hadComment = true - end := strings.Index(f.line[f.position:], f.state.block.End) - if end < 0 { - if f.state.keep { + if segment.comment { + f.hadComment = true + } + endPosition, found := segmentEnd(f.line, f.position, *segment) + if !found { + if segment.keep { f.result.WriteString(f.line[f.position:]) } f.position = len(f.line) return true } - endPosition := f.position + end + len(f.state.block.End) - if f.state.keep { + if segment.keep { f.result.WriteString(f.line[f.position:endPosition]) } f.position = endPosition - f.state.active = false - - return true -} - -// consumeActiveTextBlock copies text block content until the closing delimiter. -func (f *markerLineFilter) consumeActiveTextBlock() bool { - if !f.state.textBlockActive { - return false - } - endPosition, found := textBlockEnd(f.line, f.position, f.state.textBlock) - if !found { - f.result.WriteString(f.line[f.position:]) - f.position = len(f.line) - - return true - } - f.result.WriteString(f.line[f.position:endPosition]) - f.position = endPosition - f.state.textBlockActive = false - f.state.textBlock = TextBlockMarker{} + f.state.segment = nil return true } -// textBlockEnd returns the end offset of a text block close delimiter. -func textBlockEnd(line string, position int, marker TextBlockMarker) (int, bool) { +// segmentEnd returns the end offset of an active segment close delimiter. +func segmentEnd(line string, position int, segment activeSegment) (int, bool) { for cursor := position; cursor < len(line); { - if marker.Escapes && line[cursor] == '\\' { + if segment.escapes && line[cursor] == '\\' { cursor += 2 continue } - if strings.HasPrefix(line[cursor:], marker.Delimiter) { - return cursor + len(marker.Delimiter), true + if strings.HasPrefix(line[cursor:], segment.end) { + return cursor + len(segment.end), true } cursor++ } @@ -246,113 +149,69 @@ func (f *markerLineFilter) consumeTextBlockStart() bool { if !found { return false } - f.result.WriteString(marker.Delimiter) - f.position += len(marker.Delimiter) - f.state.textBlockActive = true - f.state.textBlock = marker - - return true -} - -// consumeQuotedSegment copies a quoted segment without scanning comment markers inside it. -func (f *markerLineFilter) consumeQuotedSegment() bool { - quoteEnd := quotedSegmentEnd(f.line, f.position, f.filter.Syntax.QuoteChars) - if quoteEnd <= f.position { - return false + f.consumeMarker(marker.Delimiter) + f.state.segment = &activeSegment{ + end: marker.Delimiter, + keep: true, + escapes: marker.Escapes, } - f.result.WriteString(f.line[f.position:quoteEnd]) - f.position = quoteEnd return true } -// quotedSegmentEnd returns the end offset of a quoted string starting at position. -func quotedSegmentEnd(line string, position int, quoteChars string) int { - if position >= len(line) || !strings.ContainsRune(quoteChars, rune(line[position])) { - return position - } - quote := line[position] - cursor := position + 1 - for cursor < len(line) { - if line[cursor] == '\\' { - cursor += 2 - - continue - } - if line[cursor] == quote { - return cursor + 1 - } - cursor++ - } - - return len(line) -} - -// consumeComment consumes a comment and reports whether it consumed input and ended the line. -func (f *markerLineFilter) consumeComment() (bool, bool) { +// consumeComment consumes a comment when one starts at the scanner position. +func (f *markerLineFilter) consumeComment() commentConsumeResult { if prefixAt(f.line, f.position, f.filter.Syntax.Documentation.Inline) { - f.consumeInlineComment(f.mode == RetainDocumentation) + f.consumeLineComment(f.mode == RetainDocumentation) - return true, true + return commentConsumeResult{consumed: true, stopLine: true} } if block, found := blockAt(f.line, f.position, f.filter.Syntax.Documentation.Block); found { f.startBlockComment(block, f.mode == RetainDocumentation) - return true, false + return commentConsumeResult{consumed: true} } if prefixAt(f.line, f.position, f.filter.Syntax.Inline) { - f.consumeInlineComment(f.mode == RetainInline || f.mode == RetainRegular) + f.consumeLineComment(f.mode == RetainInline || f.mode == RetainRegular) - return true, true + return commentConsumeResult{consumed: true, stopLine: true} } if block, found := blockAt(f.line, f.position, f.filter.Syntax.Block); found { f.startBlockComment(block, f.mode == RetainBlock || f.mode == RetainRegular) - return true, false + return commentConsumeResult{consumed: true} } - return false, false -} - -// consumeInlineComment consumes the rest of the line as a line comment. -func (f *markerLineFilter) consumeInlineComment(keep bool) { - f.hadComment = true - if keep { - f.result.WriteString(f.line[f.position:]) - } - f.position = len(f.line) + return commentConsumeResult{} } // startBlockComment records the active block comment markers and whether to keep them. func (f *markerLineFilter) startBlockComment(block BlockMarker, keep bool) { f.hadComment = true - f.state.active = true - f.state.block = block - f.state.keep = keep -} - -// consumeCodeByte copies one source byte that does not belong to a recognized comment. -func (f *markerLineFilter) consumeCodeByte() { - f.result.WriteByte(f.line[f.position]) - f.position++ + start := block.Start + if block.End == cStyleBlockCommentEnd && strings.HasPrefix(start, cStyleDocCommentStart) { + start = cStyleBlockCommentStart + } + if keep { + f.result.WriteString(start) + } + f.position += len(start) + f.state.segment = &activeSegment{ + end: block.End, + keep: keep, + comment: true, + } } // prefixAt reports whether one of the given prefixes starts at the position. func prefixAt(line string, position int, prefixes []string) bool { - _, found := prefixFrom(line, position, prefixes) - - return found -} - -// prefixFrom returns the prefix starting at position when one exists. -func prefixFrom(line string, position int, prefixes []string) (string, bool) { for _, prefix := range prefixes { if strings.HasPrefix(line[position:], prefix) { - return prefix, true + return true } } - return "", false + return false } // textBlockAt reports whether one of the given text block markers starts at the position. diff --git a/embedding/commentfilter/marker_syntax.go b/embedding/commentfilter/marker_syntax.go new file mode 100644 index 0000000..153b42e --- /dev/null +++ b/embedding/commentfilter/marker_syntax.go @@ -0,0 +1,70 @@ +// Copyright 2026, TeamDev. All rights reserved. +// +// Redistribution and use in source and/or binary forms, with or without +// modification, must retain the above copyright notice and the following +// disclaimer. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package commentfilter + +// BlockMarker describes a block comment marker pair. +type BlockMarker struct { + // Start is the block comment opening marker. + Start string + + // End is the block comment closing marker. + End string +} + +// DocumentationMarker describes API documentation comment markers. +type DocumentationMarker struct { + // Inline contains documentation line-comment markers. + Inline []string + + // Block contains documentation block-comment marker pairs. + Block []BlockMarker +} + +// TextBlockMarker describes a multi-line text literal delimiter. +type TextBlockMarker struct { + // Delimiter opens and closes the text literal. + Delimiter string + + // Escapes reports whether backslashes escape delimiter bytes. + Escapes bool +} + +// CommentMarker describes lexical comment markers and string delimiters for a language family. +type CommentMarker struct { + // Inline contains line-comment markers. + Inline []string + + // Block contains block-comment marker pairs. + Block []BlockMarker + + // Documentation contains API documentation comment markers. + Documentation DocumentationMarker + + // TextBlocks contains markers that open and close multi-line text literals. + TextBlocks []TextBlockMarker + + // QuoteChars contains characters that open and close quoted strings. + QuoteChars string +} + +// MarkerCommentFilter removes comments using lexical markers declared in CommentMarker. +type MarkerCommentFilter struct { + // Syntax contains the comment markers and string delimiters to recognize. + Syntax CommentMarker +} diff --git a/embedding/commentfilter/python_filter.go b/embedding/commentfilter/python_filter.go new file mode 100644 index 0000000..3faad20 --- /dev/null +++ b/embedding/commentfilter/python_filter.go @@ -0,0 +1,436 @@ +// Copyright 2026, TeamDev. All rights reserved. +// +// Redistribution and use in source and/or binary forms, with or without +// modification, must retain the above copyright notice and the following +// disclaimer. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package commentfilter + +import "strings" + +const ( + pythonEscapedOpenBrace = "{{" + pythonEscapedCloseBrace = "}}" +) + +// pythonInterpolation describes the Python f-string `{...}` interpolation form. +var pythonInterpolation = interpolationForm{ + start: "{", + openBrace: '{', + closeBrace: '}', +} + +// pythonStringStart describes a Python string opening token. +type pythonStringStart struct { + // token is the source text that opens the string. + token string + + // delimiter closes the string. + delimiter string + + // multiline reports whether the string can continue across source lines. + multiline bool + + // interpolated reports whether the string contains f-string expression holes. + interpolated bool +} + +// PythonCommentFilter filters Python comments while preserving string literal text. +type PythonCommentFilter struct{} + +// pythonState tracks Python lexical state that can span source lines. +type pythonState struct { + // stringActive reports whether scanning is inside string text. + stringActive bool + + // stringDelimiter closes the active string. + stringDelimiter string + + // stringMultiline reports whether the active string can continue across source lines. + stringMultiline bool + + // stringInterpolated reports whether the active string has interpolation holes. + stringInterpolated bool + + // interpolationDepth is the active brace depth inside an f-string expression. + interpolationDepth int + + // interpolationFormat reports whether scanning is inside f-string format text. + interpolationFormat bool + + // interpolationParenDepth is the parenthesis depth inside an f-string expression. + interpolationParenDepth int + + // interpolationBracketDepth is the bracket depth inside an f-string expression. + interpolationBracketDepth int + + // expressionStringActive reports whether expression scanning is inside a string. + expressionStringActive bool + + // expressionStringDelimiter closes the active expression string. + expressionStringDelimiter string + + // expressionStringMultiline reports whether the expression string can span lines. + expressionStringMultiline bool +} + +// pythonLineFilter filters one Python source line. +type pythonLineFilter struct { + // lineFilter provides shared line scanning state and helpers. + lineFilter + + // state tracks Python constructs across lines. + state *pythonState +} + +// Filter removes or preserves Python comments according to mode. +// +// Parameters: +// lines - provides Python source lines. +// mode - selects comments to retain. +// +// Returns filtered source lines. +func (PythonCommentFilter) Filter(lines []string, mode Mode) []string { + state := pythonState{} + + return filterLines(lines, func(line string) (string, bool) { + filter := pythonLineFilter{ + lineFilter: lineFilter{line: line, mode: mode}, + state: &state, + } + + return filter.filterLine() + }) +} + +// filterLine walks the current line until it reaches its end or a line comment. +func (f *pythonLineFilter) filterLine() (string, bool) { + for f.position < len(f.line) { + if f.consumeStringInterpolation() { + continue + } + if f.consumeStringText() { + continue + } + if f.consumeStringStart() { + continue + } + if f.consumeComment().consumed { + break + } + f.consumeCodeByte() + } + + f.closeSingleLineStateAtLineEnd() + + return f.result.String(), f.hadComment +} + +// consumeStringInterpolation filters code inside an active f-string expression. +func (f *pythonLineFilter) consumeStringInterpolation() bool { + if f.state.interpolationDepth == 0 { + return false + } + for f.position < len(f.line) { + if f.consumeInterpolationSegment() { + if f.state.interpolationDepth == 0 { + f.state.stringActive = true + + return true + } + + continue + } + if f.consumeInterpolationCodeByte() { + f.state.stringActive = true + + return true + } + } + + return true +} + +// consumeInterpolationSegment consumes multi-byte f-string expression content. +func (f *pythonLineFilter) consumeInterpolationSegment() bool { + if f.consumeExpressionString() { + return true + } + if f.consumeInterpolationFormat() { + return true + } + if comment := f.consumeComment(); comment.consumed { + return true + } + + return false +} + +// consumeInterpolationFormat copies f-string format text after a top-level colon. +func (f *pythonLineFilter) consumeInterpolationFormat() bool { + if !f.state.interpolationFormat { + if !f.startsFormatSpec() { + return false + } + f.state.interpolationFormat = true + f.consumeCodeByte() + } + for f.position < len(f.line) { + if f.line[f.position] == '}' { + f.consumeCodeByte() + f.closeInterpolation() + + return true + } + f.consumeCodeByte() + } + + return true +} + +// startsFormatSpec reports whether the current colon starts f-string format text. +func (f *pythonLineFilter) startsFormatSpec() bool { + return f.state.interpolationDepth == 1 && + f.state.interpolationParenDepth == 0 && + f.state.interpolationBracketDepth == 0 && + f.line[f.position] == ':' +} + +// consumeInterpolationCodeByte copies one expression byte and updates Python nesting state. +func (f *pythonLineFilter) consumeInterpolationCodeByte() bool { + char := f.line[f.position] + f.consumeCodeByte() + switch char { + case '(': + f.state.interpolationParenDepth++ + case ')': + if f.state.interpolationParenDepth > 0 { + f.state.interpolationParenDepth-- + } + case '[': + f.state.interpolationBracketDepth++ + case ']': + if f.state.interpolationBracketDepth > 0 { + f.state.interpolationBracketDepth-- + } + case pythonInterpolation.openBrace: + f.state.interpolationDepth++ + case pythonInterpolation.closeBrace: + f.state.interpolationDepth-- + if f.state.interpolationDepth == 0 { + f.closeInterpolation() + + return true + } + } + + return false +} + +// consumeExpressionString copies strings nested inside an f-string expression. +func (f *pythonLineFilter) consumeExpressionString() bool { + if f.state.expressionStringActive { + return f.consumeActiveExpressionString() + } + start, found := pythonStringStartAt(f.line, f.position) + if !found { + return false + } + f.consumeMarker(start.token) + f.state.expressionStringActive = true + f.state.expressionStringDelimiter = start.delimiter + f.state.expressionStringMultiline = start.multiline + + return true +} + +// consumeActiveExpressionString copies active expression string text. +func (f *pythonLineFilter) consumeActiveExpressionString() bool { + for f.position < len(f.line) { + switch { + case f.line[f.position] == '\\': + f.writeEscapedByte() + case f.hasPrefix(f.state.expressionStringDelimiter): + f.consumeMarker(f.state.expressionStringDelimiter) + f.closeExpressionString() + + return true + default: + f.consumeCodeByte() + } + } + + return true +} + +// consumeStringText copies active Python string text without scanning comment markers inside it. +func (f *pythonLineFilter) consumeStringText() bool { + if !f.state.stringActive || f.state.interpolationDepth > 0 { + return false + } + for f.position < len(f.line) { + if f.consumeStringTextSegment() { + return true + } + f.consumeCodeByte() + } + + return true +} + +// consumeStringTextSegment consumes special syntax inside active Python string text. +func (f *pythonLineFilter) consumeStringTextSegment() bool { + switch { + case f.line[f.position] == '\\': + f.writeEscapedByte() + case f.hasPrefix(f.state.stringDelimiter): + f.consumeMarker(f.state.stringDelimiter) + f.closeString() + case f.startsEscapedInterpolationBrace(): + f.consumeMarker(f.line[f.position : f.position+2]) + case f.state.stringInterpolated && f.line[f.position] == '{': + f.consumeCodeByte() + f.state.stringActive = false + f.state.interpolationDepth = 1 + default: + return false + } + + return true +} + +// consumeStringStart starts a Python string literal at the current position. +func (f *pythonLineFilter) consumeStringStart() bool { + start, found := pythonStringStartAt(f.line, f.position) + if !found { + return false + } + f.consumeMarker(start.token) + f.state.stringActive = true + f.state.stringDelimiter = start.delimiter + f.state.stringMultiline = start.multiline + f.state.stringInterpolated = start.interpolated + + return true +} + +// startsEscapedInterpolationBrace reports whether the position starts {{ or }} in f-string text. +func (f *pythonLineFilter) startsEscapedInterpolationBrace() bool { + if !f.state.stringInterpolated || f.position+1 >= len(f.line) { + return false + } + + return f.hasPrefix(pythonEscapedOpenBrace) || f.hasPrefix(pythonEscapedCloseBrace) +} + +// closeSingleLineStateAtLineEnd clears line-local Python string state at end of line. +func (f *pythonLineFilter) closeSingleLineStateAtLineEnd() { + if f.state.stringActive && !f.state.stringMultiline && f.state.interpolationDepth == 0 { + f.closeString() + } + if f.state.expressionStringActive && !f.state.expressionStringMultiline { + f.closeExpressionString() + } +} + +// closeString clears active Python string state. +func (f *pythonLineFilter) closeString() { + f.state.stringActive = false + f.state.stringDelimiter = "" + f.state.stringMultiline = false + f.state.stringInterpolated = false + f.closeInterpolation() +} + +// closeExpressionString clears active f-string expression string state. +func (f *pythonLineFilter) closeExpressionString() { + f.state.expressionStringActive = false + f.state.expressionStringDelimiter = "" + f.state.expressionStringMultiline = false +} + +// closeInterpolation clears active Python f-string expression state. +func (f *pythonLineFilter) closeInterpolation() { + f.state.interpolationDepth = 0 + f.state.interpolationFormat = false + f.state.interpolationParenDepth = 0 + f.state.interpolationBracketDepth = 0 +} + +// consumeComment consumes a Python line comment when one starts at the scanner position. +func (f *pythonLineFilter) consumeComment() commentConsumeResult { + if !f.hasPrefix("#") { + return commentConsumeResult{} + } + f.consumeLineComment(f.mode == RetainAll) + + return commentConsumeResult{consumed: true, stopLine: true} +} + +// pythonStringStartAt returns a Python string opener at position. +func pythonStringStartAt(line string, position int) (pythonStringStart, bool) { + for _, prefixLength := range []int{2, 1, 0} { + if position+prefixLength >= len(line) { + continue + } + prefix := line[position : position+prefixLength] + interpolated, valid := pythonStringPrefix(prefix) + if !valid { + continue + } + if delimiter, multiline, found := pythonStringDelimiterAt(line, position+prefixLength); found { + return pythonStringStart{ + token: prefix + delimiter, + delimiter: delimiter, + multiline: multiline, + interpolated: interpolated, + }, true + } + } + + return pythonStringStart{}, false +} + +// pythonStringDelimiterAt returns a Python string delimiter at position. +func pythonStringDelimiterAt(line string, position int) (string, bool, bool) { + switch { + case strings.HasPrefix(line[position:], pythonDoubleQuoteBlock): + return pythonDoubleQuoteBlock, true, true + case strings.HasPrefix(line[position:], pythonSingleQuoteBlock): + return pythonSingleQuoteBlock, true, true + case line[position] == '"': + return `"`, false, true + case line[position] == '\'': + return `'`, false, true + default: + return "", false, false + } +} + +// pythonStringPrefix reports whether prefix can precede a Python string literal. +func pythonStringPrefix(prefix string) (bool, bool) { + switch strings.ToLower(prefix) { + case "": + return false, true + case "f": + return true, true + case "r", "u", "b", "br", "rb": + return false, true + case "fr", "rf": + return true, true + default: + return false, false + } +} diff --git a/embedding/commentfilter/python_filter_test.go b/embedding/commentfilter/python_filter_test.go index ad39fc9..f5a523a 100644 --- a/embedding/commentfilter/python_filter_test.go +++ b/embedding/commentfilter/python_filter_test.go @@ -68,4 +68,48 @@ var _ = Describe("Python", func() { assertFiltered("module.py", RetainNone, lines, expected) }) + + It("should strip hash comments inside multi-line f-string expressions", func() { + lines := []string{ + "message = f\"\"\"", + "Keep this # f-string text.", + "Total: {", + " compute_total(items) # expression comment", + "}", + "\"\"\"", + `formatted = f"{value:#x}" # inline comment`, + "slice = f\"\"\"", + "{", + " values[", + " : # slice comment", + " ]", + "}", + "\"\"\"", + `braces = rf"{{ # literal }} {value # expression comment`, + `}"`, + "value = 1 # real comment", + } + + expected := []string{ + "message = f\"\"\"", + "Keep this # f-string text.", + "Total: {", + " compute_total(items) ", + "}", + "\"\"\"", + `formatted = f"{value:#x}" `, + "slice = f\"\"\"", + "{", + " values[", + " : ", + " ]", + "}", + "\"\"\"", + `braces = rf"{{ # literal }} {value `, + `}"`, + "value = 1 ", + } + + assertFiltered("module.py", RetainNone, lines, expected) + }) }) diff --git a/embedding/commentfilter/visual_basic_filter.go b/embedding/commentfilter/visual_basic_filter.go index 5478692..251add0 100644 --- a/embedding/commentfilter/visual_basic_filter.go +++ b/embedding/commentfilter/visual_basic_filter.go @@ -43,16 +43,9 @@ type VisualBasicCommentFilter struct{} // // Returns filtered source lines. func (VisualBasicCommentFilter) Filter(lines []string, mode Mode) []string { - var filtered []string - for _, line := range lines { - filteredLine, hadComment := filterVisualBasicLine(line, mode) - if hadComment && strings.TrimSpace(filteredLine) == "" { - continue - } - filtered = append(filtered, filteredLine) - } - - return filtered + return filterLines(lines, func(line string) (string, bool) { + return filterVisualBasicLine(line, mode) + }) } // filterVisualBasicLine removes or preserves one Visual Basic comment. diff --git a/showcase/embedding/positive/comment-filtering.md b/showcase/embedding/positive/comment-filtering.md index 371a840..68f4153 100644 --- a/showcase/embedding/positive/comment-filtering.md +++ b/showcase/embedding/positive/comment-filtering.md @@ -24,6 +24,9 @@ embedded unchanged. Not every supported language distinguishes documentation, regular, inline, and block comments, so unsupported categories simply have no comments to keep. +For Python f-strings, `#` text in the literal part stays unchanged, while real +`#` comments inside `{...}` expression holes are filtered. + ## Language Support | Language | Extensions | Supported `comments` modes |