Refactor comment filtering

embedding/commentfilter/c_style_filter_test.go

@@ -80,4 +80,16 @@ var _ = Describe("C and C++", func() {
 
 		assertFiltered("sample.hpp", RetainBlock, lines, expected)
 	})
+
+	It("should not close block comments on an overlapping end marker", func() {
+		lines := []string{
+			"int before = 1; /*/ still comment */ int after = 2;",
+		}
+
+		expected := []string{
+			"int before = 1;  int after = 2;",
+		}
+
+		assertFiltered("sample.cpp", RetainNone, lines, expected)
+	})
 })

embedding/commentfilter/config.go

@@ -27,7 +27,6 @@ const (
 	pythonSingleQuoteBlock  = "'''"
 	goRawStringDelimiter    = "`"
 	goQuoteChars            = "\"'"
-	jsQuoteChars            = "\"'`"
 )
 
 // filtersByExtension is a mapping of the file extension to its comment filter.
@@ -64,9 +63,9 @@ var filtersByExtension = map[string]filterEntry{
 	".proto": filterConfig(MarkerCommentFilter{Syntax: cStyleSyntax}, regularModes),
 
 	// Python
-	".py":  filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode),
-	".pyi": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode),
-	".pyw": filterConfig(MarkerCommentFilter{Syntax: pythonSyntax}, noneMode),
+	".py":  filterConfig(PythonCommentFilter{}, noneMode),
+	".pyi": filterConfig(PythonCommentFilter{}, noneMode),
+	".pyw": filterConfig(PythonCommentFilter{}, noneMode),
 
 	// YAML
 	".yml":  filterConfig(MarkerCommentFilter{Syntax: hashLineSyntax}, noneMode),
@@ -133,15 +132,6 @@ var hashLineSyntax = CommentMarker{
 	QuoteChars: "\"'",
 }
 
-var pythonSyntax = CommentMarker{
-	Inline: []string{"#"},
-	TextBlocks: []TextBlockMarker{
-		{Delimiter: pythonDoubleQuoteBlock, Escapes: true},
-		{Delimiter: pythonSingleQuoteBlock, Escapes: true},
-	},
-	QuoteChars: "\"'",
-}
-
 var xmlSyntax = CommentMarker{
 	Block: []BlockMarker{
 		{Start: "<!--", End: "-->"},