Completely migrate to prism

lib/irb.rb

@@ -6,7 +6,6 @@
 #
 
 require "prism"
-require "ripper"
 require "reline"
 
 require_relative "irb/init"
@@ -252,11 +251,10 @@ def read_input_nomultiline(prompt)
         code << line
         return code if command?(code)
 
-        tokens, opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
+        continue, opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
         return code if terminated
 
         line_offset += 1
-        continue = @scanner.should_continue?(tokens)
         prompt = generate_prompt(opens, continue, line_offset)
       end
     end
@@ -317,21 +315,25 @@ def configure_io
           else
             next true if command?(code)
 
-            _tokens, _opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
+            _continue, _opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
             terminated
           end
         end
       end
       if @context.io.respond_to?(:dynamic_prompt)
         @context.io.dynamic_prompt do |lines|
           code = lines.map{ |l| l + "\n" }.join
-          tokens = RubyLex.ripper_lex_without_warning(code, local_variables: @context.local_variables)
           parse_lex_result = Prism.parse_lex(code, scopes: [@context.local_variables])
           line_results = IRB::NestingParser.parse_by_line(parse_lex_result)
+
+          tokens = parse_lex_result.value[1].map(&:first)
+          tokens_by_line = tokens.group_by {|t| t.location.start_line - 1 }
+
           tokens_until_line = []
           line_results.map.with_index do |(_prev_opens, next_opens, _min_depth), line_num_offset|
-            tokens_until_line << tokens.shift while !tokens.empty? && tokens.first.pos[0] <= line_num_offset + 1
-            continue = @scanner.should_continue?(tokens_until_line)
+            line = lines[line_num_offset]
+            tokens_until_line.concat(tokens_by_line[line_num_offset] || [])
+            continue = @scanner.should_continue?(tokens_until_line, line, line_num_offset + 1)
             generate_prompt(next_opens, continue, line_num_offset)
           end
         end

lib/irb/command/internal_helpers.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
+require 'prism'
+
 module IRB
   module Command
     # Internal use only, for default command's backward compatibility.
     module RubyArgsExtractor # :nodoc:
       def unwrap_string_literal(str)
         return if str.empty?
 
-        sexp = Ripper.sexp(str)
-        if sexp && sexp.size == 2 && sexp.last&.first&.first == :string_literal
-          @irb_context.workspace.binding.eval(str).to_s
+        result = Prism.parse(str)
+        body = result.value.statements.body
+        if result.success? && body.size == 1 && body.first.is_a?(Prism::StringNode)
+          body.first.unescaped
         else
           str
         end

lib/irb/completion.rb

@@ -166,22 +166,13 @@ def complete_require_path(target, preposing, postposing)
       else
         return nil # It's not String literal
       end
-      tokens = RubyLex.ripper_lex_without_warning(preposing.rstrip)
-      tok = nil
-      tokens.reverse_each do |t|
-        unless [:on_lparen, :on_sp, :on_ignored_sp, :on_nl, :on_ignored_nl, :on_comment].include?(t.event)
-          tok = t
-          break
-        end
-      end
-      return unless tok&.event == :on_ident && tok.state == Ripper::EXPR_CMDARG
 
-      case tok.tok
-      when 'require'
+      case preposing
+      when /(^|[^\w])require\(? *\z/
         retrieve_files_to_require_from_load_path.filter_map { |path|
           quote + path if path.start_with?(actual_target)
         }
-      when 'require_relative'
+      when /(^|[^\w])require_relative\(? *\z/
         retrieve_files_to_require_relative_from_current_dir.filter_map { |path|
           quote + path if path.start_with?(actual_target)
         }