Importing projects site projects from .sb3 files

app/jobs/upload_job.rb

@@ -131,6 +131,11 @@ def categorize_files(files, project_dir, locale, repository, owner)
     }
 
     files.each do |file|
+      if file.extension == '.sb3'
+        categories[:components] << scratch_file_component(file, project_dir, locale, repository, owner)
+        next
+      end
+
       mime_type = file_mime_type(file)
 
       case mime_type
@@ -158,11 +163,20 @@ def component(file)
     { name:, extension:, content:, default: }
   end
 
+  def scratch_file_component(file, project_dir, locale, repository, owner)
+    name = file.name.chomp(file.extension)
+    extension = file.extension[1..]
+    { name:, extension:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open }
+  end
+
   def media(file, project_dir, locale, repository, owner)
     filename = file.name
+    { filename:, io: URI.parse(file_url(file, project_dir, locale, repository, owner)).open }
+  end
+
+  def file_url(file, project_dir, locale, repository, owner)
     directory = project_dir.name
-    url = "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/#{locale}/code/#{directory}/#{filename}"
-    { filename:, io: URI.parse(url).open }
+    "https://github.com/#{owner}/#{repository}/raw/#{ENV.fetch('GITHUB_WEBHOOK_REF')}/#{locale}/code/#{directory}/#{file.name}"
   end
 
   def repository(payload)

app/models/filesystem_project.rb

@@ -3,15 +3,16 @@
 require 'yaml'
 
 class FilesystemProject
-  CODE_FORMATS = ['.py', '.csv', '.txt', '.html', '.css'].freeze
+  CODE_FORMATS = ['.py', '.csv', '.txt', '.html', '.css', '.sb3'].freeze
   PROJECTS_ROOT = Rails.root.join('lib/tasks/project_components')
   PROJECT_CONFIG = 'project_config.yml'
 
   def self.import_all!
     PROJECTS_ROOT.each_child do |dir|
       proj_config = YAML.safe_load_file(dir.join(PROJECT_CONFIG).to_s)
 
-      files = dir.children.reject { |file| file.basename.to_s == 'project_config.yml' }
+      files = dir.children.reject { |file| file.basename.to_s == PROJECT_CONFIG }
+      files = configured_scratch_files(files, proj_config) if proj_config['TYPE'] == Project::Types::CODE_EDITOR_SCRATCH
       categorized_files = categorize_files(files, dir)
 
       project_importer = ProjectImporter.new(name: proj_config['NAME'], identifier: proj_config['IDENTIFIER'],
@@ -53,9 +54,18 @@ def self.categorize_files(files, dir)
     categories
   end
 
+  def self.configured_scratch_files(files, proj_config)
+    configured_locations = Array(proj_config['COMPONENTS']).pluck('location')
+    return files if configured_locations.empty?
+
+    files.reject { |file| File.extname(file) == '.sb3' && configured_locations.exclude?(file.basename.to_s) }
+  end
+
   def self.component(file, dir)
     name = File.basename(file, '.*')
     extension = File.extname(file).delete('.')
+    return { name:, extension:, file_path: dir.join(File.basename(file)).to_s } if extension == 'sb3'
+
     code = File.read(dir.join(File.basename(file)).to_s)
     default = (File.basename(file) == 'main.py')
     { name:, extension:, content: code, default: }

lib/project_importer.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ProjectImporter
+  class ImportError < StandardError; end
+
   attr_reader :name, :identifier, :images, :videos, :audio, :media, :components, :type, :locale
 
   def initialize(**kwargs)
@@ -20,6 +22,8 @@ def import!
       setup_project
       delete_components
       create_components
+      create_scratch_component
+      create_scratch_assets
       delete_removed_media
       attach_media_if_needed
 
@@ -39,16 +43,47 @@ def setup_project
   end
 
   def delete_components
+    return if project.scratch_project?
+
     project.components.each(&:destroy)
   end
 
   def create_components
+    return if project.scratch_project?
+
     components.each do |component|
+      # .sb3 files are only ever imported as a ScratchComponent (see
+      # create_scratch_component); they carry an :io/:file_path key that is not a
+      # Component attribute, so skip them here to avoid building invalid rows.
+      next if component[:extension] == 'sb3'
+
       project_component = Component.new(**component)
       project.components << project_component
     end
   end
 
+  def create_scratch_component
+    return unless project.scratch_project?
+
+    component = components[0]
+    return unless component&.fetch(:extension, nil) == 'sb3'
+
+    parsed_content = Sb3Parser.new(component: component).parse.fetch(:scratch_component).fetch(:content)
+    raise ImportError, 'Scratch project content could not be parsed' if parsed_content.blank?
+
+    project.scratch_component = ScratchComponent.new(content: parsed_content)
+  end
+
+  def create_scratch_assets
+    return unless project.scratch_project?
+
+    component = components[0]
+    return unless component&.fetch(:extension, nil) == 'sb3'
+
+    parsed_assets = Sb3Parser.new(component: component).parse.fetch(:assets)
+    ScratchSb3AssetImporter.import_all(parsed_assets)
+  end
+
   def delete_removed_media
     return if removed_media_names.empty?
 

lib/sb3_parser.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'marcel'
+require 'stringio'
+require 'zip'
+
+class Sb3Parser
+  class MissingProjectJsonError < StandardError; end
+  class MissingAssetError < StandardError; end
+
+  attr_reader :component, :file_path, :io
+
+  def initialize(component: nil, file_path: nil)
+    @component = component
+    @file_path = component&.fetch(:file_path, nil) || file_path
+    @io = component&.fetch(:io, nil)
+  end
+
+  def parse
+    open_zip do |zip_file|
+      project_json = project_json_entry(zip_file)
+      content = JSON.parse(project_json.get_input_stream.read)
+
+      output = {
+        scratch_component: { content: },
+        assets: assets(zip_file, extract_asset_names(content))
+      }
+      output
+    end
+  end
+
+  private
+
+  def open_zip(&)
+    return Zip::File.open(file_path, &) if file_path
+
+    io.rewind if io.respond_to?(:rewind)
+    result = nil
+    Zip::File.open_buffer(io.read) { |zip_file| result = yield zip_file }
+    result
+  end
+
+  def project_json_entry(zip_file)
+    zip_file.find_entry('project.json') || raise(MissingProjectJsonError, 'project.json not found in SB3 archive')
+  end
+
+  def extract_asset_names(value)
+    case value
+    when Hash
+      names = []
+      names << value['md5ext'] if value['md5ext'].is_a?(String)
+      value.each_value { |item| names.concat(extract_asset_names(item)) }
+      names.uniq
+    when Array
+      value.flat_map { |item| extract_asset_names(item) }.uniq
+    else
+      []
+    end
+  end
+
+  def assets(zip_file, asset_names)
+    asset_names.map do |asset_name|
+      entry = zip_file.find_entry(asset_name) || raise(MissingAssetError, "asset #{asset_name} not found in SB3 archive")
+      asset(entry)
+    end
+  end
+
+  def asset(entry)
+    io = StringIO.new(entry.get_input_stream.read)
+    content_type = Marcel::MimeType.for(io, name: entry.name)
+    io.rewind
+
+    { filename: entry.name, io:, content_type: }
+  end
+end

lib/scratch_asset_importer.rb

@@ -45,6 +45,8 @@ def asset
     end
   end
 
+  private
+
   def create_scratch_asset
     return if ScratchAsset.global_assets.exists?(filename: asset_name)
 

lib/scratch_sb3_asset_importer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+class ScratchSb3AssetImporter
+  class << self
+    def import_all(assets)
+      assets.each do |asset|
+        new.import(asset)
+      rescue StandardError
+        next
+      end
+    end
+  end
+
+  def import(asset)
+    create_asset(asset.fetch(:filename), asset.fetch(:io).read)
+  end
+
+  private
+
+  def create_asset(asset_name, content)
+    return if ScratchAsset.global_assets.exists?(filename: asset_name)
+
+    ScratchAsset.create!(filename: asset_name, project_id: nil, uploaded_user_id: nil)
+                .file
+                .attach(io: StringIO.new(content), filename: asset_name)
+  end
+end

lib/tasks/project_components/scratch-integration-test-starter/project_config.yml

@@ -0,0 +1,3 @@
+NAME: "scratch integration test"
+IDENTIFIER: "editor-scratch-testing-starter"
+TYPE: "code_editor_scratch"

spec/jobs/upload_job_spec.rb

@@ -269,6 +269,116 @@
     end
   end
 
+  context 'when a scratch project is uploaded' do
+    let(:scratch_payload) do
+      {
+        repository: { name: 'my-amazing-repo', owner: { name: 'me' } },
+        commits: [{ added: ['ja-JP/code/scratch-integration-test-starter/main.sb3'], modified: [], removed: [] }]
+      }
+    end
+    let(:scratch_project_json) do
+      {
+        targets: [
+          {
+            costumes: [{ md5ext: 'test_image_1.png' }],
+            sounds: [{ md5ext: 'test_audio_1.mp3' }]
+          }
+        ]
+      }
+    end
+    let(:scratch_sb3_body) do
+      sb3_archive_string(
+        'project.json' => scratch_project_json.to_json,
+        'test_image_1.png' => sb3_fixture_content('test_image_1.png'),
+        'test_audio_1.mp3' => sb3_fixture_content('test_audio_1.mp3')
+      )
+    end
+    let(:raw_response) do
+      {
+        data: {
+          repository: {
+            object: {
+              __typename: 'Tree',
+              entries: [
+                {
+                  name: 'scratch-integration-test-starter',
+                  object: {
+                    __typename: 'Tree',
+                    entries: [
+                      {
+                        name: 'main.sb3',
+                        extension: '.sb3',
+                        object: {
+                          __typename: 'Blob',
+                          text: nil,
+                          isBinary: true
+                        }
+                      },
+                      {
+                        name: 'project_config.yml',
+                        extension: '.yml',
+                        object: {
+                          __typename: 'Blob',
+                          text: "name: \"Scratch Integration Test\"\nidentifier: \"scratch-integration-test-starter\"\ntype: \"code_editor_scratch\"\n",
+                          isBinary: false
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            }
+          }
+        }
+      }.deep_stringify_keys
+    end
+
+    before do
+      allow(GithubApi::Client).to receive(:query).and_return(graphql_response)
+      allow(ProjectImporter).to receive(:new).and_call_original
+
+      stub_request(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/scratch-integration-test-starter/main.sb3')
+        .to_return(status: 200, body: scratch_sb3_body, headers: {})
+    end
+
+    it 'imports the Scratch project with the sb3 component as io' do
+      described_class.perform_now(scratch_payload)
+
+      expect(ProjectImporter).to have_received(:new).with(
+        hash_including(
+          name: 'Scratch Integration Test',
+          identifier: 'scratch-integration-test-starter',
+          type: Project::Types::CODE_EDITOR_SCRATCH,
+          locale: 'ja-JP',
+          images: [],
+          videos: [],
+          audio: [],
+          components: [
+            hash_including(
+              name: 'main',
+              extension: 'sb3',
+              io: an_object_responding_to(:read)
+            )
+          ]
+        )
+      )
+    end
+
+    it 'requests the sb3 file from the correct URL' do
+      described_class.perform_now(scratch_payload)
+
+      expect(WebMock).to have_requested(:get, 'https://github.com/me/my-amazing-repo/raw/branches/whatever/ja-JP/code/scratch-integration-test-starter/main.sb3').once
+    end
+
+    it 'saves the Scratch project to the database' do
+      expect { described_class.perform_now(scratch_payload) }.to change(Project, :count).by(1)
+
+      project = Project.find_by(identifier: 'scratch-integration-test-starter', locale: 'ja-JP')
+      expect(project.project_type).to eq(Project::Types::CODE_EDITOR_SCRATCH)
+      expect(project.scratch_component.content).to eq(JSON.parse(scratch_project_json.to_json))
+    end
+  end
+
   context 'when locale is unsupported' do
     let(:raw_response) { { data: { repository: nil } } }
     let(:bad_payload) do