From 2dadc30db2ed7515eb3938aa6695c077337c8f09 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 5 Jun 2026 13:55:56 +0100 Subject: [PATCH 01/29] add starter yml --- .../scratch-integration-test-starter/project_config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 lib/tasks/project_components/scratch-integration-test-starter/project_config.yml diff --git a/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml b/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml new file mode 100644 index 000000000..a1a0201ed --- /dev/null +++ b/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml @@ -0,0 +1,9 @@ +NAME: "scratch integration test" +IDENTIFIER: "editor-scratch-testing-starter" +TYPE: "code_editor_scratch" +COMPONENTS: + - name: "main" + extension: "sb3" + location: "main.sb3" + index: 0 + default: true From dfe4b87aa921fc049c1555b4e1dc870ef8a17d70 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 5 Jun 2026 13:57:14 +0100 Subject: [PATCH 02/29] allow processing sb3 files --- app/models/filesystem_project.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/filesystem_project.rb b/app/models/filesystem_project.rb index 3632ba3e2..29848710b 100644 --- a/app/models/filesystem_project.rb +++ b/app/models/filesystem_project.rb @@ -3,7 +3,7 @@ require 'yaml' class FilesystemProject - CODE_FORMATS = ['.py', '.csv', '.txt', '.html', '.css'].freeze + CODE_FORMATS = ['.py', '.csv', '.txt', '.html', '.css', '.sb3'].freeze PROJECTS_ROOT = Rails.root.join('lib/tasks/project_components') PROJECT_CONFIG = 'project_config.yml' From 2c317825195b90ca382b4aa282c519d3aceaa1a7 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 5 Jun 2026 13:57:32 +0100 Subject: [PATCH 03/29] initial parser attempt --- lib/sb3_parser.rb | 68 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 lib/sb3_parser.rb diff --git a/lib/sb3_parser.rb b/lib/sb3_parser.rb new file mode 100644 index 000000000..c81127d42 --- /dev/null +++ b/lib/sb3_parser.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'json' +require 'marcel' +require 'stringio' +require 'zip' + +class Sb3Parser + class MissingProjectJsonError < StandardError; end + class MissingAssetError < StandardError; end + + attr_reader :file_path + + def initialize(file_path:) + @file_path = file_path + end + + def parse + Zip::File.open(file_path) do |zip_file| + project_json = project_json_entry(zip_file) + content = JSON.parse(project_json.get_input_stream.read) + + output = { + scratch_component: { content: } + # assets: assets(zip_file, extract_asset_names(content)) + } + pp output + output + end + end + + private + + def project_json_entry(zip_file) + zip_file.find_entry('project.json') || raise(MissingProjectJsonError, 'project.json not found in SB3 archive') + end + + # def extract_asset_names(value) + # case value + # when Hash + # names = [] + # names << value['md5ext'] if value['md5ext'].is_a?(String) + # value.each_value { |item| names.concat(extract_asset_names(item)) } + # names.uniq + # when Array + # value.flat_map { |item| extract_asset_names(item) }.uniq + # else + # [] + # end + # end + + # def assets(zip_file, asset_names) + # entries_by_name = zip_file.each.reject(&:directory?).index_by { |entry| entry.name } + + # asset_names.map do |asset_name| + # entry = entries_by_name[asset_name] || raise(MissingAssetError, "asset #{asset_name} not found in SB3 archive") + # asset(entry) + # end + # end + + # def asset(entry) + # io = StringIO.new(entry.get_input_stream.read) + # content_type = Marcel::MimeType.for(io, name: entry.name) + # io.rewind + + # { filename: entry.name, io:, content_type: } + # end +end From 2277ac07a31654c87916f8a78b7dba3ac8652ad4 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 11:21:44 +0100 Subject: [PATCH 04/29] update parser to work with GH webhook --- lib/sb3_parser.rb | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/sb3_parser.rb b/lib/sb3_parser.rb index c81127d42..173f2a888 100644 --- a/lib/sb3_parser.rb +++ b/lib/sb3_parser.rb @@ -9,14 +9,16 @@ class Sb3Parser class MissingProjectJsonError < StandardError; end class MissingAssetError < StandardError; end - attr_reader :file_path + attr_reader :component, :file_path, :io - def initialize(file_path:) - @file_path = file_path + def initialize(component: nil, file_path: nil) + @component = component + @file_path = component&.fetch(:file_path, nil) || file_path + @io = component&.fetch(:io, nil) end def parse - Zip::File.open(file_path) do |zip_file| + open_zip do |zip_file| project_json = project_json_entry(zip_file) content = JSON.parse(project_json.get_input_stream.read) @@ -31,6 +33,15 @@ def parse private + def open_zip + return Zip::File.open(file_path) { |zip_file| yield zip_file } if file_path + + io.rewind if io.respond_to?(:rewind) + result = nil + Zip::File.open_buffer(io.read) { |zip_file| result = yield zip_file } + result + end + def project_json_entry(zip_file) zip_file.find_entry('project.json') || raise(MissingProjectJsonError, 'project.json not found in SB3 archive') end From e539c07ecc04ab5c5c4968d8f19d86e426ef866c Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 11:22:21 +0100 Subject: [PATCH 05/29] implement scratch importing in project importer --- lib/project_importer.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 2669336ba..c16ef51cc 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -20,6 +20,7 @@ def import! setup_project delete_components create_components + create_scratch_component delete_removed_media attach_media_if_needed @@ -39,16 +40,31 @@ def setup_project end def delete_components + return unless project.project_type != 'code_editor_scratch' + project.components.each(&:destroy) end def create_components + return unless project.project_type != 'code_editor_scratch' + components.each do |component| project_component = Component.new(**component) project.components << project_component end end + def create_scratch_component + return unless project.project_type == 'code_editor_scratch' + + components.each do |component| + next unless component[:extension] == 'sb3' + + parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) + project.scratch_component = ScratchComponent.new(content: parsed_content) if parsed_content + end + end + def delete_removed_media return if removed_media_names.empty? From f0584784f41867c37244d027673897c7308339f7 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 11:26:38 +0100 Subject: [PATCH 06/29] update upload job to accept and process sb3 files --- app/jobs/upload_job.rb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/app/jobs/upload_job.rb b/app/jobs/upload_job.rb index a72de493f..65fa4bd6c 100644 --- a/app/jobs/upload_job.rb +++ b/app/jobs/upload_job.rb @@ -131,6 +131,11 @@ def categorize_files(files, project_dir, locale, repository, owner) } files.each do |file| + if file.extension == '.sb3' + categories[:components] << component(file, project_dir, locale, repository, owner) + next + end + mime_type = file_mime_type(file) case mime_type @@ -150,9 +155,11 @@ def categorize_files(files, project_dir, locale, repository, owner) categories end - def component(file) + def component(file, project_dir = nil, locale = nil, repository = nil, owner = nil) name = file.name.chomp(file.extension) extension = file.extension[1..] + return { name:, extension:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } if extension == 'sb3' + content = file.object.text default = file.name == 'main.py' { name:, extension:, content:, default: } @@ -160,9 +167,12 @@ def component(file) def media(file, project_dir, locale, repository, owner) filename = file.name + { filename:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } + end + + def file_url(file, project_dir, locale, repository, owner) directory = project_dir.name - url = "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/#{locale}/code/#{directory}/#{filename}" - { filename:, io: URI.parse(url).open } + "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/#{locale}/code/#{directory}/#{file.name}" end def repository(payload) From 29d0a7a09b8ce4e49e44f72de7250a4c78d9a5ba Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 11:43:20 +0100 Subject: [PATCH 07/29] update filesystem_project for sb3 support --- app/models/filesystem_project.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/models/filesystem_project.rb b/app/models/filesystem_project.rb index 29848710b..906e43a3e 100644 --- a/app/models/filesystem_project.rb +++ b/app/models/filesystem_project.rb @@ -11,7 +11,8 @@ def self.import_all! PROJECTS_ROOT.each_child do |dir| proj_config = YAML.safe_load_file(dir.join(PROJECT_CONFIG).to_s) - files = dir.children.reject { |file| file.basename.to_s == 'project_config.yml' } + files = dir.children.reject { |file| file.basename.to_s == PROJECT_CONFIG } + files = configured_scratch_files(files, dir, proj_config) if proj_config['TYPE'] == Project::Types::CODE_EDITOR_SCRATCH categorized_files = categorize_files(files, dir) project_importer = ProjectImporter.new(name: proj_config['NAME'], identifier: proj_config['IDENTIFIER'], @@ -53,9 +54,18 @@ def self.categorize_files(files, dir) categories end + def self.configured_scratch_files(files, dir, proj_config) + configured_locations = Array(proj_config['COMPONENTS']).pluck('location') + return files if configured_locations.empty? + + files.reject { |file| File.extname(file) == '.sb3' && configured_locations.exclude?(file.basename.to_s) } + end + def self.component(file, dir) name = File.basename(file, '.*') extension = File.extname(file).delete('.') + return { name:, extension:, file_path: dir.join(File.basename(file)).to_s } if extension == 'sb3' + code = File.read(dir.join(File.basename(file)).to_s) default = (File.basename(file) == 'main.py') { name:, extension:, content: code, default: } From fdc972801481ba65dc01dc6f9e7209e85664b031 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Mon, 8 Jun 2026 15:15:40 +0100 Subject: [PATCH 08/29] tiny clean up on sb3 parser --- lib/sb3_parser.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/sb3_parser.rb b/lib/sb3_parser.rb index 173f2a888..aa0c5ffd2 100644 --- a/lib/sb3_parser.rb +++ b/lib/sb3_parser.rb @@ -26,15 +26,14 @@ def parse scratch_component: { content: } # assets: assets(zip_file, extract_asset_names(content)) } - pp output output end end private - def open_zip - return Zip::File.open(file_path) { |zip_file| yield zip_file } if file_path + def open_zip(&) + return Zip::File.open(file_path, &) if file_path io.rewind if io.respond_to?(:rewind) result = nil From bddaab629958eb40585ddf18c4cc6d0af923b986 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Tue, 9 Jun 2026 11:42:58 +0100 Subject: [PATCH 09/29] add asset importing for sb3 files --- lib/scratch_asset_importer.rb | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index 632fba835..0d3694ea4 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -22,7 +22,11 @@ def show_progress? end end - attr_reader :asset_base_url, :asset_name + def self.import_from_sb3_assets(assets, asset_base_url) + new(nil, asset_base_url).import_from_sb3_assets(assets) + end + + attr_reader :asset_base_url, :asset_names ASSET_FETCHING_DELAY = 0.2 @@ -45,7 +49,18 @@ def asset end end - def create_scratch_asset + def import_from_sb3_assets(assets) + bar = ProgressBar.create(format: '%t: |%B| %c of %C %E', total: assets.count) if show_progress? + + assets.each do |asset| + bar.increment if show_progress? + import_sb3_asset(asset.fetch(:filename), asset.fetch(:io).read) + end + end + + private + + def create_scratch_asset(asset_names) return if ScratchAsset.global_assets.exists?(filename: asset_name) io = StringIO.new(asset.body) @@ -99,6 +114,17 @@ def s3_client ) end + def import_sb3_asset(asset_name, content) + return if ScratchAsset.global_assets.exists?(filename: asset_name) + + sleep(ASSET_FETCHING_DELAY) + ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil) + .file + .attach(io: StringIO.new(content), filename: asset_name) + rescue StandardError => e + Rails.logger.error("Failed to import SB3 asset #{asset_name}: #{e.message}") + end + def connection @connection ||= Faraday.new(url: asset_base_url) do |faraday| faraday.response :raise_error From 42c2ef2d9da663da316e40ece538d80691be6dd7 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Tue, 9 Jun 2026 11:43:21 +0100 Subject: [PATCH 10/29] update sb3 parser to pull files from zip --- lib/sb3_parser.rb | 56 +++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/lib/sb3_parser.rb b/lib/sb3_parser.rb index aa0c5ffd2..0dd14b4e9 100644 --- a/lib/sb3_parser.rb +++ b/lib/sb3_parser.rb @@ -23,8 +23,8 @@ def parse content = JSON.parse(project_json.get_input_stream.read) output = { - scratch_component: { content: } - # assets: assets(zip_file, extract_asset_names(content)) + scratch_component: { content: }, + assets: assets(zip_file, extract_asset_names(content)) } output end @@ -45,34 +45,32 @@ def project_json_entry(zip_file) zip_file.find_entry('project.json') || raise(MissingProjectJsonError, 'project.json not found in SB3 archive') end - # def extract_asset_names(value) - # case value - # when Hash - # names = [] - # names << value['md5ext'] if value['md5ext'].is_a?(String) - # value.each_value { |item| names.concat(extract_asset_names(item)) } - # names.uniq - # when Array - # value.flat_map { |item| extract_asset_names(item) }.uniq - # else - # [] - # end - # end - - # def assets(zip_file, asset_names) - # entries_by_name = zip_file.each.reject(&:directory?).index_by { |entry| entry.name } + def extract_asset_names(value) + case value + when Hash + names = [] + names << value['md5ext'] if value['md5ext'].is_a?(String) + value.each_value { |item| names.concat(extract_asset_names(item)) } + names.uniq + when Array + value.flat_map { |item| extract_asset_names(item) }.uniq + else + [] + end + end - # asset_names.map do |asset_name| - # entry = entries_by_name[asset_name] || raise(MissingAssetError, "asset #{asset_name} not found in SB3 archive") - # asset(entry) - # end - # end + def assets(zip_file, asset_names) + asset_names.map do |asset_name| + entry = zip_file.find_entry(asset_name) || raise(MissingAssetError, "asset #{asset_name} not found in SB3 archive") + asset(entry) + end + end - # def asset(entry) - # io = StringIO.new(entry.get_input_stream.read) - # content_type = Marcel::MimeType.for(io, name: entry.name) - # io.rewind + def asset(entry) + io = StringIO.new(entry.get_input_stream.read) + content_type = Marcel::MimeType.for(io, name: entry.name) + io.rewind - # { filename: entry.name, io:, content_type: } - # end + { filename: entry.name, io:, content_type: } + end end From e63e015abfdc0e64f89ecfd2ffeb81b2765d7771 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Tue, 9 Jun 2026 11:43:34 +0100 Subject: [PATCH 11/29] call asset importer in project importer --- lib/project_importer.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index c16ef51cc..ea0994e4d 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -21,6 +21,7 @@ def import! delete_components create_components create_scratch_component + create_scratch_assets delete_removed_media attach_media_if_needed @@ -60,11 +61,22 @@ def create_scratch_component components.each do |component| next unless component[:extension] == 'sb3' - parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) + parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component) project.scratch_component = ScratchComponent.new(content: parsed_content) if parsed_content end end + def create_scratch_assets + return unless project.project_type == 'code_editor_scratch' + + components.each do |component| + next unless component[:extension] == 'sb3' + + parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) + ScratchAssetImporter.import_from_sb3_assets(parsed_assets, 'test.com') + end + end + def delete_removed_media return if removed_media_names.empty? From 5a4c68bf3d99dbb79403443f70288591c327e012 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Tue, 9 Jun 2026 15:45:26 +0100 Subject: [PATCH 12/29] fix import bug --- lib/project_importer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index ea0994e4d..88c72c4b9 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -61,7 +61,7 @@ def create_scratch_component components.each do |component| next unless component[:extension] == 'sb3' - parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component) + parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) project.scratch_component = ScratchComponent.new(content: parsed_content) if parsed_content end end From 676f9fe77d42a1f2a4641f6720c6ba148fc23df5 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 11:44:22 +0100 Subject: [PATCH 13/29] refactor asset importer for consistency with original importer --- lib/project_importer.rb | 2 +- lib/scratch_asset_importer.rb | 43 +++++++++++++++++------------------ 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 88c72c4b9..73f8b2c2d 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -73,7 +73,7 @@ def create_scratch_assets next unless component[:extension] == 'sb3' parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) - ScratchAssetImporter.import_from_sb3_assets(parsed_assets, 'test.com') + ScratchAssetImporter.import_from_sb3(parsed_assets) end end diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index 0d3694ea4..aea7cc0d5 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -14,6 +14,13 @@ def import_all(asset_names, asset_base_url) new(asset_name, asset_base_url).import end end + + def import_from_sb3(assets) + + assets.each do |asset| + new(nil, nil).import_from_sb3(asset) + end + end private @@ -22,11 +29,7 @@ def show_progress? end end - def self.import_from_sb3_assets(assets, asset_base_url) - new(nil, asset_base_url).import_from_sb3_assets(assets) - end - - attr_reader :asset_base_url, :asset_names + attr_reader :asset_base_url, :asset_name ASSET_FETCHING_DELAY = 0.2 @@ -49,13 +52,8 @@ def asset end end - def import_from_sb3_assets(assets) - bar = ProgressBar.create(format: '%t: |%B| %c of %C %E', total: assets.count) if show_progress? - - assets.each do |asset| - bar.increment if show_progress? - import_sb3_asset(asset.fetch(:filename), asset.fetch(:io).read) - end + def import_from_sb3(asset) + create_sb3_asset(asset.fetch(:filename), asset.fetch(:io).read) end private @@ -70,6 +68,17 @@ def create_scratch_asset(asset_names) .attach(io:, filename: asset_name) end + def create_sb3_asset(asset_name, content) + return if ScratchAsset.global_assets.exists?(filename: asset_name) + + sleep(ASSET_FETCHING_DELAY) + ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil) + .file + .attach(io: StringIO.new(content), filename: asset_name) + rescue StandardError => e + Rails.logger.error("Failed to import SB3 asset #{asset_name}: #{e.message}") + end + def save_to_editor_asset_bucket return unless save_to_editor_asset_bucket? @@ -114,16 +123,6 @@ def s3_client ) end - def import_sb3_asset(asset_name, content) - return if ScratchAsset.global_assets.exists?(filename: asset_name) - - sleep(ASSET_FETCHING_DELAY) - ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil) - .file - .attach(io: StringIO.new(content), filename: asset_name) - rescue StandardError => e - Rails.logger.error("Failed to import SB3 asset #{asset_name}: #{e.message}") - end def connection @connection ||= Faraday.new(url: asset_base_url) do |faraday| From 888151ca19082c52e045013819b726fc058a98ce Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 12:10:24 +0100 Subject: [PATCH 14/29] clean up --- lib/scratch_asset_importer.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index aea7cc0d5..0abe6e699 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -14,9 +14,8 @@ def import_all(asset_names, asset_base_url) new(asset_name, asset_base_url).import end end - + def import_from_sb3(assets) - assets.each do |asset| new(nil, nil).import_from_sb3(asset) end @@ -58,7 +57,7 @@ def import_from_sb3(asset) private - def create_scratch_asset(asset_names) + def create_scratch_asset return if ScratchAsset.global_assets.exists?(filename: asset_name) io = StringIO.new(asset.body) @@ -123,7 +122,6 @@ def s3_client ) end - def connection @connection ||= Faraday.new(url: asset_base_url) do |faraday| faraday.response :raise_error From d9eed537009ff499b69f15a7bf2aba3e49e4a696 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 14:12:56 +0100 Subject: [PATCH 15/29] separate scratch component in upload job --- app/jobs/upload_job.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/jobs/upload_job.rb b/app/jobs/upload_job.rb index 65fa4bd6c..53eb178b8 100644 --- a/app/jobs/upload_job.rb +++ b/app/jobs/upload_job.rb @@ -132,7 +132,7 @@ def categorize_files(files, project_dir, locale, repository, owner) files.each do |file| if file.extension == '.sb3' - categories[:components] << component(file, project_dir, locale, repository, owner) + categories[:components] << scratch_file_component(file, project_dir, locale, repository, owner) next end @@ -155,16 +155,20 @@ def categorize_files(files, project_dir, locale, repository, owner) categories end - def component(file, project_dir = nil, locale = nil, repository = nil, owner = nil) + def component(file) name = file.name.chomp(file.extension) extension = file.extension[1..] - return { name:, extension:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } if extension == 'sb3' - content = file.object.text default = file.name == 'main.py' { name:, extension:, content:, default: } end + def scratch_file_component(file, project_dir, locale, repository, owner) + name = file.name.chomp(file.extension) + extension = file.extension[1..] + { name:, extension:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } + end + def media(file, project_dir, locale, repository, owner) filename = file.name { filename:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open } From a6ac0ac2bd1d939770719b66d7e32079407d866b Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 15:40:55 +0100 Subject: [PATCH 16/29] rubocop --- app/models/filesystem_project.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/filesystem_project.rb b/app/models/filesystem_project.rb index 906e43a3e..aa5ce5753 100644 --- a/app/models/filesystem_project.rb +++ b/app/models/filesystem_project.rb @@ -12,7 +12,7 @@ def self.import_all! proj_config = YAML.safe_load_file(dir.join(PROJECT_CONFIG).to_s) files = dir.children.reject { |file| file.basename.to_s == PROJECT_CONFIG } - files = configured_scratch_files(files, dir, proj_config) if proj_config['TYPE'] == Project::Types::CODE_EDITOR_SCRATCH + files = configured_scratch_files(files, proj_config) if proj_config['TYPE'] == Project::Types::CODE_EDITOR_SCRATCH categorized_files = categorize_files(files, dir) project_importer = ProjectImporter.new(name: proj_config['NAME'], identifier: proj_config['IDENTIFIER'], @@ -54,7 +54,7 @@ def self.categorize_files(files, dir) categories end - def self.configured_scratch_files(files, dir, proj_config) + def self.configured_scratch_files(files, proj_config) configured_locations = Array(proj_config['COMPONENTS']).pluck('location') return files if configured_locations.empty? From 5c0c65a401e8364315b73b2f3285d1659df16165 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 11 Jun 2026 16:20:59 +0100 Subject: [PATCH 17/29] add sb3 parser tests --- spec/lib/sb3_parser_spec.rb | 84 ++++++++++++++++++++++++++++++ spec/support/sb3_archive_helper.rb | 24 +++++++++ 2 files changed, 108 insertions(+) create mode 100644 spec/lib/sb3_parser_spec.rb create mode 100644 spec/support/sb3_archive_helper.rb diff --git a/spec/lib/sb3_parser_spec.rb b/spec/lib/sb3_parser_spec.rb new file mode 100644 index 000000000..492172d96 --- /dev/null +++ b/spec/lib/sb3_parser_spec.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'sb3_parser' + +RSpec.describe Sb3Parser do + describe '#parse' do + let(:png_content) { sb3_fixture_content('test_image_1.png') } + let(:mp3_content) { sb3_fixture_content('test_audio_1.mp3') } + let(:project_json) do + { + targets: [ + { + costumes: [ + { name: 'cat', md5ext: 'abc123.png' }, + { name: 'duplicate cat', md5ext: 'abc123.png' } + ], + sounds: [ + { name: 'meow', md5ext: 'def456.mp3' } + ] + } + ] + } + end + let(:entries) do + { + 'project.json' => project_json.to_json, + 'abc123.png' => png_content, + 'def456.mp3' => mp3_content + } + end + + it 'parses project.json and referenced assets from component io' do + result = described_class.new(component: { io: sb3_archive(entries) }).parse + + expect(result.fetch(:scratch_component).fetch(:content)).to eq(JSON.parse(project_json.to_json)) + + assets = result.fetch(:assets) + expect(assets.map { |asset| asset.fetch(:filename) }).to contain_exactly('abc123.png', 'def456.mp3') + + png_asset = assets.find { |asset| asset.fetch(:filename) == 'abc123.png' } + expect(png_asset.fetch(:content_type)).to eq('image/png') + expect(png_asset.fetch(:io).read).to eq(png_content) + end + + it 'parses an archive from a file path' do + Tempfile.create(['scratch-project', '.sb3']) do |file| + archive = sb3_archive(entries) + file.binmode + file.write(archive.read) + file.flush + + result = described_class.new(file_path: file.path).parse + + expect(result.fetch(:scratch_component).fetch(:content)).to eq(JSON.parse(project_json.to_json)) + expect(result.fetch(:assets).map { |asset| asset.fetch(:filename) }).to contain_exactly('abc123.png', 'def456.mp3') + end + end + + it 'returns no assets when project.json does not reference any md5ext values' do + archive = sb3_archive('project.json' => { targets: [] }.to_json) + + result = described_class.new(component: { io: archive }).parse + + expect(result.fetch(:assets)).to eq([]) + end + + it 'raises when project.json is missing' do + archive = sb3_archive('abc123.png' => png_content) + + expect do + described_class.new(component: { io: archive }).parse + end.to raise_error(described_class::MissingProjectJsonError, 'project.json not found in SB3 archive') + end + + it 'raises when a referenced asset is missing' do + archive = sb3_archive('project.json' => { targets: [{ costumes: [{ md5ext: 'missing.png' }] }] }.to_json) + + expect do + described_class.new(component: { io: archive }).parse + end.to raise_error(described_class::MissingAssetError, 'asset missing.png not found in SB3 archive') + end + end +end diff --git a/spec/support/sb3_archive_helper.rb b/spec/support/sb3_archive_helper.rb new file mode 100644 index 000000000..4c920b642 --- /dev/null +++ b/spec/support/sb3_archive_helper.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Sb3ArchiveHelper + def sb3_archive(entries) + Zip::OutputStream.write_buffer do |zip| + entries.each do |name, content| + zip.put_next_entry(name) + zip.write(content) + end + end.tap(&:rewind) + end + + def sb3_archive_string(entries) + sb3_archive(entries).string + end + + def sb3_fixture_content(filename) + Rails.root.join('spec/fixtures/files', filename).binread + end +end + +RSpec.configure do |config| + config.include Sb3ArchiveHelper +end From 5ec7c7227db9edc94cd2c732b8dd61124db0b0e0 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 12 Jun 2026 08:38:33 +0100 Subject: [PATCH 18/29] clear up confusing naming --- lib/project_importer.rb | 2 +- lib/scratch_asset_importer.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 73f8b2c2d..d04216ee0 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -73,7 +73,7 @@ def create_scratch_assets next unless component[:extension] == 'sb3' parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) - ScratchAssetImporter.import_from_sb3(parsed_assets) + ScratchAssetImporter.import_all_from_sb3(parsed_assets) end end diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index 0abe6e699..e9a98449d 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -15,7 +15,7 @@ def import_all(asset_names, asset_base_url) end end - def import_from_sb3(assets) + def import_all_from_sb3(assets) assets.each do |asset| new(nil, nil).import_from_sb3(asset) end From 6ae3fd725751d0c825e963a35f3da3f499ad8c1c Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Fri, 12 Jun 2026 08:39:43 +0100 Subject: [PATCH 19/29] update tests for sb3 import functionality --- spec/jobs/upload_job_spec.rb | 110 ++++++++++++++++++++++++ spec/lib/project_importer_spec.rb | 101 ++++++++++++++++++++++ spec/lib/scratch_asset_importer_spec.rb | 59 +++++++++++++ 3 files changed, 270 insertions(+) diff --git a/spec/jobs/upload_job_spec.rb b/spec/jobs/upload_job_spec.rb index 61d5bb86a..be69fe875 100644 --- a/spec/jobs/upload_job_spec.rb +++ b/spec/jobs/upload_job_spec.rb @@ -269,6 +269,116 @@ end end + context 'when a scratch project is uploaded' do + let(:scratch_payload) do + { + repository: { name: 'my-amazing-repo', owner: { name: 'me' } }, + commits: [{ added: ['ja-JP/code/scratch-integration-test-starter/main.sb3'], modified: [], removed: [] }] + } + end + let(:scratch_project_json) do + { + targets: [ + { + costumes: [{ md5ext: 'test_image_1.png' }], + sounds: [{ md5ext: 'test_audio_1.mp3' }] + } + ] + } + end + let(:scratch_sb3_body) do + sb3_archive_string( + 'project.json' => scratch_project_json.to_json, + 'test_image_1.png' => sb3_fixture_content('test_image_1.png'), + 'test_audio_1.mp3' => sb3_fixture_content('test_audio_1.mp3') + ) + end + let(:raw_response) do + { + data: { + repository: { + object: { + __typename: 'Tree', + entries: [ + { + name: 'scratch-integration-test-starter', + object: { + __typename: 'Tree', + entries: [ + { + name: 'main.sb3', + extension: '.sb3', + object: { + __typename: 'Blob', + text: nil, + isBinary: true + } + }, + { + name: 'project_config.yml', + extension: '.yml', + object: { + __typename: 'Blob', + text: "name: \"Scratch Integration Test\"\nidentifier: \"scratch-integration-test-starter\"\ntype: \"code_editor_scratch\"\n", + isBinary: false + } + } + ] + } + } + ] + } + } + } + }.deep_stringify_keys + end + + before do + allow(GithubApi::Client).to receive(:query).and_return(graphql_response) + allow(ProjectImporter).to receive(:new).and_call_original + + stub_request(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/scratch-integration-test-starter/main.sb3') + .to_return(status: 200, body: scratch_sb3_body, headers: {}) + end + + it 'imports the Scratch project with the sb3 component as io' do + described_class.perform_now(scratch_payload) + + expect(ProjectImporter).to have_received(:new).with( + hash_including( + name: 'Scratch Integration Test', + identifier: 'scratch-integration-test-starter', + type: Project::Types::CODE_EDITOR_SCRATCH, + locale: 'ja-JP', + images: [], + videos: [], + audio: [], + components: [ + hash_including( + name: 'main', + extension: 'sb3', + io: an_object_responding_to(:read) + ) + ] + ) + ) + end + + it 'requests the sb3 file from the correct URL' do + described_class.perform_now(scratch_payload) + + expect(WebMock).to have_requested(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/scratch-integration-test-starter/main.sb3').once + end + + it 'saves the Scratch project to the database' do + expect { described_class.perform_now(scratch_payload) }.to change(Project, :count).by(1) + + project = Project.find_by(identifier: 'scratch-integration-test-starter', locale: 'ja-JP') + expect(project.project_type).to eq(Project::Types::CODE_EDITOR_SCRATCH) + expect(project.scratch_component.content).to eq(JSON.parse(scratch_project_json.to_json)) + end + end + context 'when locale is unsupported' do let(:raw_response) { { data: { repository: nil } } } let(:bad_payload) do diff --git a/spec/lib/project_importer_spec.rb b/spec/lib/project_importer_spec.rb index 7d847dc54..f38380543 100644 --- a/spec/lib/project_importer_spec.rb +++ b/spec/lib/project_importer_spec.rb @@ -114,4 +114,105 @@ expect { importer.import! }.to change { project.reload.audio[0].filename.to_s }.to('my-amazing-audio.mp3') end end + + context 'when the project has type code_editor_scratch' do + let(:scratch_project_file) { Tempfile.new(['test_scratch_project', '.sb3']) } + let(:importer) do + described_class.new( + name: 'My amazing Scratch project', + identifier: 'my-amazing-scratch-project', + type: Project::Types::CODE_EDITOR_SCRATCH, + locale: 'en', + components: [ + { name: 'main', extension: 'sb3', file_path: scratch_project_file.path } + ] + ) + end + let(:scratch_project_content) do + { + targets: [ + { + costumes: [{ md5ext: 'test_image_1.png' }], + sounds: [{ md5ext: 'test_audio_1.mp3' }], + videos: [{ md5ext: 'test_video_1.mp4' }] + } + ] + } + end + + let(:project) { Project.find_by(identifier: importer.identifier, user_id: nil, locale: importer.locale) } + + before do + scratch_project_file.binmode + scratch_project_file.write( + sb3_archive( + 'project.json' => scratch_project_content.to_json, + 'test_image_1.png' => sb3_fixture_content('test_image_1.png'), + 'test_video_1.mp4' => sb3_fixture_content('test_video_1.mp4'), + 'test_audio_1.mp3' => sb3_fixture_content('test_audio_1.mp3') + ).read + ) + scratch_project_file.flush + end + + after do + scratch_project_file.close + scratch_project_file.unlink + end + + context 'when importing a new scratch project' do + it 'imports the Scratch component content' do + importer.import! + + expect(project.components.count).to eq(0) + expect(project.scratch_component.content).to eq(JSON.parse(scratch_project_content.to_json)) + end + + it 'imports the project assets' do + importer.import! + expect(ScratchAsset.global_assets.where(filename: ['test_image_1.png', 'test_video_1.mp4', 'test_audio_1.mp3']).count).to eq(3) + end + end + + context 'when the scratch project already exists in the database' do + let(:original_scratch_content) do + { targets: ['old target'], monitors: [], extensions: [], meta: {} } + end + let!(:project) do + create( + :project, + identifier: 'my-amazing-scratch-project', + locale: 'en', + project_type: Project::Types::CODE_EDITOR_SCRATCH, + name: 'Old Scratch project name' + ) + end + + before do + create(:scratch_component, project:, content: original_scratch_content) + end + + it 'does not create a new project' do + expect { importer.import! }.not_to change(Project, :count) + end + + it 'updates the project name' do + expect { importer.import! }.to change { project.reload.name }.to(importer.name) + end + + it 'updates the scratch component content' do + importer.import! + + expect(project.reload.scratch_component.content).to eq(JSON.parse(scratch_project_content.to_json)) + end + + it 'imports any new assets without duplicating existing ones' do + create(:scratch_asset, :with_file, filename: 'test_image_1.png') + + importer.import! + + expect(ScratchAsset.global_assets.where(filename: ['test_image_1.png', 'test_video_1.mp4', 'test_audio_1.mp3']).count).to eq(3) + end + end + end end diff --git a/spec/lib/scratch_asset_importer_spec.rb b/spec/lib/scratch_asset_importer_spec.rb index d8e209c2c..9cfedd2c6 100644 --- a/spec/lib/scratch_asset_importer_spec.rb +++ b/spec/lib/scratch_asset_importer_spec.rb @@ -118,4 +118,63 @@ end end end + + describe '.import_all_from_sb3' do + def sb3_asset(filename, content = sb3_fixture_content(filename)) + { filename:, io: StringIO.new(content) } + end + + it 'imports assets from SB3 archive content' do + png_content = sb3_fixture_content('test_image_1.png') + + described_class.import_all_from_sb3([sb3_asset('test_image_1.png', png_content)]) + + scratch_asset = ScratchAsset.find_by(filename: 'test_image_1.png') + expect(scratch_asset).to be_present + expect(scratch_asset).to be_global + expect(scratch_asset.file.download).to eq(png_content) + end + + it 'does nothing if global asset already exists' do + create(:scratch_asset, :with_file, filename: 'test_image_1.png') + + expect do + described_class.import_all_from_sb3([sb3_asset('test_image_1.png')]) + end.not_to change(ScratchAsset, :count) + end + + it 'can import multiple assets' do + described_class.import_all_from_sb3([ + sb3_asset('test_image_1.png'), + sb3_asset('test_audio_1.mp3', sb3_fixture_content('test_audio_1.mp3')) + ]) + + expect(ScratchAsset.find_by(filename: 'test_image_1.png')).to be_present + expect(ScratchAsset.find_by(filename: 'test_audio_1.mp3')).to be_present + end + + it 'still imports a global asset when a project asset already uses the filename' do + project = create(:project, project_type: Project::Types::CODE_EDITOR_SCRATCH, locale: nil, user_id: SecureRandom.uuid) + create(:scratch_component, project:) + create(:scratch_asset, :with_file, filename: 'test_image_1.png', project:) + + expect do + described_class.import_all_from_sb3([sb3_asset('test_image_1.png')]) + end.to change { ScratchAsset.global_assets.where(filename: 'test_image_1.png').count }.by(1) + end + + it 'skips assets that fail to import' do + allow(ScratchAsset).to receive(:create!).and_call_original + allow(ScratchAsset).to receive(:create!).with(filename: 'failing.png', project_id: nil, uploaded_user_id: nil) + .and_raise(StandardError, 'attach failed') + + described_class.import_all_from_sb3([ + sb3_asset('failing.png', 'bad'), + sb3_asset('test_image_1.png') + ]) + + expect(ScratchAsset.find_by(filename: 'failing.png')).not_to be_present + expect(ScratchAsset.find_by(filename: 'test_image_1.png')).to be_present + end + end end From 806e5e171f3c6e12eccdb5b2886ff91766b9cb1c Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Wed, 17 Jun 2026 13:55:57 +0100 Subject: [PATCH 20/29] separate sb3 asset importer to it's own file --- lib/project_importer.rb | 2 +- lib/scratch_asset_importer.rb | 21 ------- lib/scratch_sb3_asset_importer.rb | 29 ++++++++++ spec/lib/scratch_asset_importer_spec.rb | 59 ------------------- spec/lib/scratch_sb3_asset_importer_spec.rb | 64 +++++++++++++++++++++ 5 files changed, 94 insertions(+), 81 deletions(-) create mode 100644 lib/scratch_sb3_asset_importer.rb create mode 100644 spec/lib/scratch_sb3_asset_importer_spec.rb diff --git a/lib/project_importer.rb b/lib/project_importer.rb index d04216ee0..be2af76c6 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -73,7 +73,7 @@ def create_scratch_assets next unless component[:extension] == 'sb3' parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) - ScratchAssetImporter.import_all_from_sb3(parsed_assets) + ScratchSb3AssetImporter.import_all(parsed_assets) end end diff --git a/lib/scratch_asset_importer.rb b/lib/scratch_asset_importer.rb index e9a98449d..8cf5cee21 100644 --- a/lib/scratch_asset_importer.rb +++ b/lib/scratch_asset_importer.rb @@ -15,12 +15,6 @@ def import_all(asset_names, asset_base_url) end end - def import_all_from_sb3(assets) - assets.each do |asset| - new(nil, nil).import_from_sb3(asset) - end - end - private def show_progress? @@ -51,10 +45,6 @@ def asset end end - def import_from_sb3(asset) - create_sb3_asset(asset.fetch(:filename), asset.fetch(:io).read) - end - private def create_scratch_asset @@ -67,17 +57,6 @@ def create_scratch_asset .attach(io:, filename: asset_name) end - def create_sb3_asset(asset_name, content) - return if ScratchAsset.global_assets.exists?(filename: asset_name) - - sleep(ASSET_FETCHING_DELAY) - ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil) - .file - .attach(io: StringIO.new(content), filename: asset_name) - rescue StandardError => e - Rails.logger.error("Failed to import SB3 asset #{asset_name}: #{e.message}") - end - def save_to_editor_asset_bucket return unless save_to_editor_asset_bucket? diff --git a/lib/scratch_sb3_asset_importer.rb b/lib/scratch_sb3_asset_importer.rb new file mode 100644 index 000000000..e18865fe7 --- /dev/null +++ b/lib/scratch_sb3_asset_importer.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'stringio' + +class ScratchSb3AssetImporter + class << self + def import_all(assets) + assets.each do |asset| + new.import(asset) + rescue StandardError + next + end + end + end + + def import(asset) + create_asset(asset.fetch(:filename), asset.fetch(:io).read) + end + + private + + def create_asset(asset_name, content) + return if ScratchAsset.global_assets.exists?(filename: asset_name) + + ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil) + .file + .attach(io: StringIO.new(content), filename: asset_name) + end +end diff --git a/spec/lib/scratch_asset_importer_spec.rb b/spec/lib/scratch_asset_importer_spec.rb index 9cfedd2c6..d8e209c2c 100644 --- a/spec/lib/scratch_asset_importer_spec.rb +++ b/spec/lib/scratch_asset_importer_spec.rb @@ -118,63 +118,4 @@ end end end - - describe '.import_all_from_sb3' do - def sb3_asset(filename, content = sb3_fixture_content(filename)) - { filename:, io: StringIO.new(content) } - end - - it 'imports assets from SB3 archive content' do - png_content = sb3_fixture_content('test_image_1.png') - - described_class.import_all_from_sb3([sb3_asset('test_image_1.png', png_content)]) - - scratch_asset = ScratchAsset.find_by(filename: 'test_image_1.png') - expect(scratch_asset).to be_present - expect(scratch_asset).to be_global - expect(scratch_asset.file.download).to eq(png_content) - end - - it 'does nothing if global asset already exists' do - create(:scratch_asset, :with_file, filename: 'test_image_1.png') - - expect do - described_class.import_all_from_sb3([sb3_asset('test_image_1.png')]) - end.not_to change(ScratchAsset, :count) - end - - it 'can import multiple assets' do - described_class.import_all_from_sb3([ - sb3_asset('test_image_1.png'), - sb3_asset('test_audio_1.mp3', sb3_fixture_content('test_audio_1.mp3')) - ]) - - expect(ScratchAsset.find_by(filename: 'test_image_1.png')).to be_present - expect(ScratchAsset.find_by(filename: 'test_audio_1.mp3')).to be_present - end - - it 'still imports a global asset when a project asset already uses the filename' do - project = create(:project, project_type: Project::Types::CODE_EDITOR_SCRATCH, locale: nil, user_id: SecureRandom.uuid) - create(:scratch_component, project:) - create(:scratch_asset, :with_file, filename: 'test_image_1.png', project:) - - expect do - described_class.import_all_from_sb3([sb3_asset('test_image_1.png')]) - end.to change { ScratchAsset.global_assets.where(filename: 'test_image_1.png').count }.by(1) - end - - it 'skips assets that fail to import' do - allow(ScratchAsset).to receive(:create!).and_call_original - allow(ScratchAsset).to receive(:create!).with(filename: 'failing.png', project_id: nil, uploaded_user_id: nil) - .and_raise(StandardError, 'attach failed') - - described_class.import_all_from_sb3([ - sb3_asset('failing.png', 'bad'), - sb3_asset('test_image_1.png') - ]) - - expect(ScratchAsset.find_by(filename: 'failing.png')).not_to be_present - expect(ScratchAsset.find_by(filename: 'test_image_1.png')).to be_present - end - end end diff --git a/spec/lib/scratch_sb3_asset_importer_spec.rb b/spec/lib/scratch_sb3_asset_importer_spec.rb new file mode 100644 index 000000000..10fdd1c81 --- /dev/null +++ b/spec/lib/scratch_sb3_asset_importer_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'rails_helper' +require 'scratch_sb3_asset_importer' + +RSpec.describe ScratchSb3AssetImporter do + describe '.import_all' do + def sb3_asset(filename, content = sb3_fixture_content(filename)) + { filename:, io: StringIO.new(content) } + end + + it 'imports assets from SB3 archive content' do + png_content = sb3_fixture_content('test_image_1.png') + + described_class.import_all([sb3_asset('test_image_1.png', png_content)]) + + scratch_asset = ScratchAsset.find_by(filename: 'test_image_1.png') + expect(scratch_asset).to be_present + expect(scratch_asset).to be_global + expect(scratch_asset.file.download).to eq(png_content) + end + + it 'does nothing if global asset already exists' do + create(:scratch_asset, :with_file, filename: 'test_image_1.png') + + expect do + described_class.import_all([sb3_asset('test_image_1.png')]) + end.not_to change(ScratchAsset, :count) + end + + it 'can import multiple assets' do + described_class.import_all([ + sb3_asset('test_image_1.png'), + sb3_asset('test_audio_1.mp3', sb3_fixture_content('test_audio_1.mp3')) + ]) + + expect(ScratchAsset.find_by(filename: 'test_image_1.png')).to be_present + expect(ScratchAsset.find_by(filename: 'test_audio_1.mp3')).to be_present + end + + it 'still imports a global asset when a project asset already uses the filename' do + project = create(:project, project_type: Project::Types::CODE_EDITOR_SCRATCH, locale: nil, user_id: SecureRandom.uuid) + create(:scratch_component, project:) + create(:scratch_asset, :with_file, filename: 'test_image_1.png', project:) + + expect do + described_class.import_all([sb3_asset('test_image_1.png')]) + end.to change { ScratchAsset.global_assets.where(filename: 'test_image_1.png').count }.by(1) + end + + it 'skips assets that fail to import' do + allow(ScratchAsset).to receive(:create!).and_call_original + allow(ScratchAsset).to receive(:create!).with(filename: 'failing.png', project_id: nil, uploaded_user_id: nil) + + described_class.import_all([ + sb3_asset('failing.png', 'bad'), + sb3_asset('test_image_1.png') + ]) + + expect(ScratchAsset.find_by(filename: 'failing.png')).not_to be_present + expect(ScratchAsset.find_by(filename: 'test_image_1.png')).to be_present + end + end +end From e2fe5e37afe4acae1dbbf188c35af546b5530d7a Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 18 Jun 2026 10:39:49 +0100 Subject: [PATCH 21/29] default to importing first component item for scratch projects --- lib/project_importer.rb | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index be2af76c6..4b833d4fa 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -58,23 +58,21 @@ def create_components def create_scratch_component return unless project.project_type == 'code_editor_scratch' - components.each do |component| - next unless component[:extension] == 'sb3' + component = components[0] + return unless component[:extension] == 'sb3' - parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) - project.scratch_component = ScratchComponent.new(content: parsed_content) if parsed_content - end + parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) + project.scratch_component = ScratchComponent.new(content: parsed_content) if parsed_content end def create_scratch_assets return unless project.project_type == 'code_editor_scratch' - components.each do |component| - next unless component[:extension] == 'sb3' + component = components[0] + return unless component[:extension] == 'sb3' - parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) - ScratchSb3AssetImporter.import_all(parsed_assets) - end + parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) + ScratchSb3AssetImporter.import_all(parsed_assets) end def delete_removed_media From d13114c204f4b3700d3db6e95b8afdb41b375a68 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 18 Jun 2026 10:40:08 +0100 Subject: [PATCH 22/29] scratch project config yaml doesn't need components section --- .../scratch-integration-test-starter/project_config.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml b/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml index a1a0201ed..f6d08cbe2 100644 --- a/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml +++ b/lib/tasks/project_components/scratch-integration-test-starter/project_config.yml @@ -1,9 +1,3 @@ NAME: "scratch integration test" IDENTIFIER: "editor-scratch-testing-starter" TYPE: "code_editor_scratch" -COMPONENTS: - - name: "main" - extension: "sb3" - location: "main.sb3" - index: 0 - default: true From 9b5337518fdcf577fcaf6da1feeaf0712e5f5446 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 18 Jun 2026 11:00:44 +0100 Subject: [PATCH 23/29] handle empty parsed_content for sb3 file --- lib/project_importer.rb | 6 +++++- spec/lib/project_importer_spec.rb | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 4b833d4fa..74cbb1d86 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ProjectImporter + class ImportError < StandardError; end + attr_reader :name, :identifier, :images, :videos, :audio, :media, :components, :type, :locale def initialize(**kwargs) @@ -62,7 +64,9 @@ def create_scratch_component return unless component[:extension] == 'sb3' parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) - project.scratch_component = ScratchComponent.new(content: parsed_content) if parsed_content + raise ImportError, 'Scratch project content could not be parsed' unless parsed_content.present? + + project.scratch_component = ScratchComponent.new(content: parsed_content) end def create_scratch_assets diff --git a/spec/lib/project_importer_spec.rb b/spec/lib/project_importer_spec.rb index f38380543..d8a9f37dd 100644 --- a/spec/lib/project_importer_spec.rb +++ b/spec/lib/project_importer_spec.rb @@ -172,6 +172,19 @@ importer.import! expect(ScratchAsset.global_assets.where(filename: ['test_image_1.png', 'test_video_1.mp4', 'test_audio_1.mp3']).count).to eq(3) end + + it 'raises and rolls back the import when the scratch content cannot be parsed' do + allow_any_instance_of(Sb3Parser).to receive(:parse).and_return({ scratch_component: { content: nil }, assets: [] }) + + expect { importer.import! } + .to raise_error(ProjectImporter::ImportError, 'Scratch project content could not be parsed') + + expect do + importer.import! + rescue StandardError + ProjectImporter::ImportError + end.not_to change(Project, :count) + end end context 'when the scratch project already exists in the database' do @@ -213,6 +226,16 @@ expect(ScratchAsset.global_assets.where(filename: ['test_image_1.png', 'test_video_1.mp4', 'test_audio_1.mp3']).count).to eq(3) end + + it 'rolls back project changes when the scratch content cannot be parsed' do + allow_any_instance_of(Sb3Parser).to receive(:parse).and_return({ scratch_component: { content: nil }, assets: [] }) + + expect { importer.import! } + .to raise_error(ProjectImporter::ImportError, 'Scratch project content could not be parsed') + + expect(project.reload.name).to eq('Old Scratch project name') + expect(project.scratch_component.content).to eq(original_scratch_content.deep_stringify_keys) + end end end end From 70dfe53a90d3ca257de11f9d0e3096115c18b09a Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 18 Jun 2026 11:07:15 +0100 Subject: [PATCH 24/29] fix tests + rubocop --- lib/project_importer.rb | 2 +- spec/lib/project_importer_spec.rb | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 74cbb1d86..dd7efbb60 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -64,7 +64,7 @@ def create_scratch_component return unless component[:extension] == 'sb3' parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) - raise ImportError, 'Scratch project content could not be parsed' unless parsed_content.present? + raise ImportError, 'Scratch project content could not be parsed' if parsed_content.blank? project.scratch_component = ScratchComponent.new(content: parsed_content) end diff --git a/spec/lib/project_importer_spec.rb b/spec/lib/project_importer_spec.rb index d8a9f37dd..da753811a 100644 --- a/spec/lib/project_importer_spec.rb +++ b/spec/lib/project_importer_spec.rb @@ -117,6 +117,13 @@ context 'when the project has type code_editor_scratch' do let(:scratch_project_file) { Tempfile.new(['test_scratch_project', '.sb3']) } + let(:parser) { instance_double(Sb3Parser, parse: parser_result) } + let(:parser_result) do + { + scratch_component: { content: JSON.parse(scratch_project_content.to_json) }, + assets: [] + } + end let(:importer) do described_class.new( name: 'My amazing Scratch project', @@ -143,6 +150,8 @@ let(:project) { Project.find_by(identifier: importer.identifier, user_id: nil, locale: importer.locale) } before do + allow(Sb3Parser).to receive(:new).and_return(parser) + scratch_project_file.binmode scratch_project_file.write( sb3_archive( @@ -174,16 +183,12 @@ end it 'raises and rolls back the import when the scratch content cannot be parsed' do - allow_any_instance_of(Sb3Parser).to receive(:parse).and_return({ scratch_component: { content: nil }, assets: [] }) + allow(parser).to receive(:parse).and_return({ scratch_component: { content: nil }, assets: [] }) expect { importer.import! } .to raise_error(ProjectImporter::ImportError, 'Scratch project content could not be parsed') - expect do - importer.import! - rescue StandardError - ProjectImporter::ImportError - end.not_to change(Project, :count) + expect(Project.where(identifier: importer.identifier, locale: importer.locale).count).to eq(0) end end @@ -228,7 +233,7 @@ end it 'rolls back project changes when the scratch content cannot be parsed' do - allow_any_instance_of(Sb3Parser).to receive(:parse).and_return({ scratch_component: { content: nil }, assets: [] }) + allow(parser).to receive(:parse).and_return({ scratch_component: { content: nil }, assets: [] }) expect { importer.import! } .to raise_error(ProjectImporter::ImportError, 'Scratch project content could not be parsed') From 91419933d1222a0c301ff6c4242e7fc413201c85 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 18 Jun 2026 12:10:34 +0100 Subject: [PATCH 25/29] fix tests --- spec/lib/project_importer_spec.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/lib/project_importer_spec.rb b/spec/lib/project_importer_spec.rb index da753811a..ab236b506 100644 --- a/spec/lib/project_importer_spec.rb +++ b/spec/lib/project_importer_spec.rb @@ -121,7 +121,11 @@ let(:parser_result) do { scratch_component: { content: JSON.parse(scratch_project_content.to_json) }, - assets: [] + assets: [ + { filename: 'test_image_1.png', io: StringIO.new(sb3_fixture_content('test_image_1.png')), content_type: 'image/png' }, + { filename: 'test_video_1.mp4', io: StringIO.new(sb3_fixture_content('test_video_1.mp4')), content_type: 'video/mp4' }, + { filename: 'test_audio_1.mp3', io: StringIO.new(sb3_fixture_content('test_audio_1.mp3')), content_type: 'audio/mpeg' } + ] } end let(:importer) do From ea51c47e9e27b25df767e44242c711b951d6212d Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 18 Jun 2026 12:20:30 +0100 Subject: [PATCH 26/29] add sb3 file for sample project --- lib/project_importer.rb | 4 ++-- .../scratch-integration-test-starter/main.sb3 | Bin 0 -> 2414 bytes 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 lib/tasks/project_components/scratch-integration-test-starter/main.sb3 diff --git a/lib/project_importer.rb b/lib/project_importer.rb index dd7efbb60..42804e488 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -61,7 +61,7 @@ def create_scratch_component return unless project.project_type == 'code_editor_scratch' component = components[0] - return unless component[:extension] == 'sb3' + return unless component&.fetch(:extension, nil) == 'sb3' parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content) raise ImportError, 'Scratch project content could not be parsed' if parsed_content.blank? @@ -73,7 +73,7 @@ def create_scratch_assets return unless project.project_type == 'code_editor_scratch' component = components[0] - return unless component[:extension] == 'sb3' + return unless component&.fetch(:extension, nil) == 'sb3' parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets) ScratchSb3AssetImporter.import_all(parsed_assets) diff --git a/lib/tasks/project_components/scratch-integration-test-starter/main.sb3 b/lib/tasks/project_components/scratch-integration-test-starter/main.sb3 new file mode 100644 index 0000000000000000000000000000000000000000..9f2148b52b610344a4732c4e73f0e80784277c50 GIT binary patch literal 2414 zcma)8c{CL28y<{pEQyhQhKej92cl=lssO-+SKko%cKEdCqyx`JU&z*5*t=5C8yR1LXMBUws1p zk~t3q0D`#y0M6srYb1X!0zOE=E70Gs2JMTU*Wfvl`%u0VZSP|?K=l~rKM5vZlrL|A z2pEbKj;Xd0K}DpfeM^wEr=!bITu%_eKYU101C$4opV?6~W06GRQu-p@i z?}Y3bvGo_7Cy*!Nz^6Vu>}J`Mtd>w^jjTH&wKNXR4NDxmNm#u01Z2BzGT3@J z$R{vbJ~HmkH=(Eq+dm_er-)%;BVWI*&K$KvWan#Aq-R(_&dciXT5a82=!eES{NRK> z)X@zH!YlxDfkp?yxX$Sx-soET{v8_zf^;=egpv{R|k+mGvW;N=ppHyt0bQu#;axG%G7* zh3#uA@N#7mYjmd{h@NkIrKQlaSP6H&Kf3~O;mn}Q($>dEV42HOYPkU3@R&aDb_nk| z5J>js?b|PNRLjXkuw*9aKbMpatPQ+{PR-1A{@%1i- z&WcI0meQ`L6*E32?F$8So++qBckhnE-}OW26>lf3IP$VJEgc+mtq&y$PcNj7terG^ zr&Tj2i*e4!?zT8r(4ebxW;CUth?#30i6F+oIBOTGGVWwkTX7YUxh*S`6I3#CYo)() z=f^BbzFJCi#Y>l4nxn#a)w!T8B;n`s$k>>%o%dJ z@EPj9i4jRtviT{f&~K>Qex06&Ei>AU3WU17@f7tR&c1Jk0!g!yBEXw=lIx}Q#uZpi%46cjx z%EkWGBwqEFuDq-@Gg^%(*ol;4JJe&!Rk=P&KYjEmb6 zUoElW@JT%^WzJ1LK!uM z_X{-hdABtqEfdZUt2(N7e@;3@=6F?9K1n^oR>vzRtnR^4yUSMVupMBC*FZ57Z zFBeQ_et%) zJ?WHf>8?ca7nx+R=PTN*H|UczAZa_2`D0Z4Y8zgUQ-U@eCDLmj##j!#RyQVRc@kqy zWHy36>u?sy4%ql~8W;EdK`r>!{}x5(cbG;q0sy7QA^%I1Iu_}Mb#q4$2v}7mbtIgCS4G0qaY}e3 zl7K*}D*Y0bY&Y#U4daZML9+?Ne4f-HV!QrXbTy|5`dsq>N#k9dd^~BJEpOLpXR0&hzN%XJtS2 zf*i4ANW^vvmV9GFqN#SD?U;5uru9sPsxTB1=$1JwBeyc_Xv~=;3s1vf--6pZ1?)BN zKP_5m@@*Pk@tnAr2o*A!wUUOyrScY><@|M%(J#%Zr}^pDqPQLw<+5iw2U?F;0>`5i zE2SZX+xqcHo?_jYktgDmu<)TnGP_=(5@L)bkjeABOHS;8#%A3*>_A@KJ1)SO#0sQusR|byBqAI>NDdv ze;*-jehJl(_WhyBq8s0%z7@69e8X$@9%HPK>RrFVXbui3zF4AP3`q(f$6C{PeUAOT zT*;PG;d5Ia7QUE(;B7>j_pY=G#DuG_hp(EKi(v5qX05#G*>A`@K*2Le?YY=#^|%_H(LQ zJ>ZTMN;+(F%bm8{li^pPV+-u*5TOUIxcXd0JKIh^mnU&iWa?^=QYv!^AB!Kzg_-wH ze(@S^6pAwnpPdq!;sR+K$l5X8&~qXekxDxF21+2MqprqmwcZ^i9Qybv6Q*+7iys-R zKodSukS9`I<)ffjlg*q(_s``9C;H0@=;QJA&gOk4YjXxhFyR019>>4)ALpN&$nVJC qo%%P7lli~-_IJkLZSptc+%e;SF^#o3Gs`a&({X(~zQfsmP5%Rf^(i|5 literal 0 HcmV?d00001 From 4bfe526d5e74e456501bf4bc2392da9f07fcf087 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 18 Jun 2026 13:53:55 +0100 Subject: [PATCH 27/29] fix return conditions on project_importer --- lib/project_importer.rb | 8 ++++---- spec/lib/project_importer_spec.rb | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index 42804e488..a2f9f94ff 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -43,13 +43,13 @@ def setup_project end def delete_components - return unless project.project_type != 'code_editor_scratch' + return if project.scratch_project? project.components.each(&:destroy) end def create_components - return unless project.project_type != 'code_editor_scratch' + return if project.scratch_project? components.each do |component| project_component = Component.new(**component) @@ -58,7 +58,7 @@ def create_components end def create_scratch_component - return unless project.project_type == 'code_editor_scratch' + return unless project.scratch_project? component = components[0] return unless component&.fetch(:extension, nil) == 'sb3' @@ -70,7 +70,7 @@ def create_scratch_component end def create_scratch_assets - return unless project.project_type == 'code_editor_scratch' + return unless project.scratch_project? component = components[0] return unless component&.fetch(:extension, nil) == 'sb3' diff --git a/spec/lib/project_importer_spec.rb b/spec/lib/project_importer_spec.rb index ab236b506..68a48b335 100644 --- a/spec/lib/project_importer_spec.rb +++ b/spec/lib/project_importer_spec.rb @@ -214,6 +214,12 @@ create(:scratch_component, project:, content: original_scratch_content) end + it 'does not delete existing standard components' do + create(:component, project:, name: 'legacy', extension: 'txt', content: 'keep me') + + expect { importer.import! }.not_to change { project.reload.components.count } + end + it 'does not create a new project' do expect { importer.import! }.not_to change(Project, :count) end From 13b859fe45fecd84f2b10a6a20713962d34f378d Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 18 Jun 2026 13:57:36 +0100 Subject: [PATCH 28/29] add require zip for sb3 archive helper --- spec/support/sb3_archive_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/support/sb3_archive_helper.rb b/spec/support/sb3_archive_helper.rb index 4c920b642..3250df52d 100644 --- a/spec/support/sb3_archive_helper.rb +++ b/spec/support/sb3_archive_helper.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'zip' module Sb3ArchiveHelper def sb3_archive(entries) Zip::OutputStream.write_buffer do |zip| From 8bb3e8566c1bc3cbf1be97c0aa79beaf7c98c0c3 Mon Sep 17 00:00:00 2001 From: Ram Modhvadia Date: Thu, 18 Jun 2026 14:06:08 +0100 Subject: [PATCH 29/29] skip sb3 component import for non scratch projects --- lib/project_importer.rb | 5 +++++ spec/lib/project_importer_spec.rb | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/lib/project_importer.rb b/lib/project_importer.rb index a2f9f94ff..350583683 100644 --- a/lib/project_importer.rb +++ b/lib/project_importer.rb @@ -52,6 +52,11 @@ def create_components return if project.scratch_project? components.each do |component| + # .sb3 files are only ever imported as a ScratchComponent (see + # create_scratch_component); they carry an :io/:file_path key that is not a + # Component attribute, so skip them here to avoid building invalid rows. + next if component[:extension] == 'sb3' + project_component = Component.new(**component) project.components << project_component end diff --git a/spec/lib/project_importer_spec.rb b/spec/lib/project_importer_spec.rb index 68a48b335..a029127a3 100644 --- a/spec/lib/project_importer_spec.rb +++ b/spec/lib/project_importer_spec.rb @@ -115,6 +115,31 @@ end end + context 'when a non-scratch project contains an sb3 file' do + let(:project) { Project.find_by(identifier: importer.identifier, user_id: nil, locale: importer.locale) } + let(:importer) do + described_class.new( + name: 'My amazing project', + identifier: 'my-amazing-project', + type: Project::Types::PYTHON, + locale: 'en', + components: [ + { name: 'main', extension: 'py', content: 'print(\'hello\')', default: true }, + { name: 'stray', extension: 'sb3', io: StringIO.new('ignored') } + ] + ) + end + + it 'does not raise when importing' do + expect { importer.import! }.not_to raise_error + end + + it 'skips the sb3 file and only creates the standard components' do + importer.import! + expect(project.components.pluck(:extension)).to eq(['py']) + end + end + context 'when the project has type code_editor_scratch' do let(:scratch_project_file) { Tempfile.new(['test_scratch_project', '.sb3']) } let(:parser) { instance_double(Sb3Parser, parse: parser_result) }