diff --git a/analysis/reanalyze/src/annotation.ml b/analysis/reanalyze/src/annotation.ml index 806d9db1f2..9e11949b09 100644 --- a/analysis/reanalyze/src/annotation.ml +++ b/analysis/reanalyze/src/annotation.ml @@ -61,7 +61,8 @@ let rec get_attribute_payload check_text (attributes : Typedtree.attributes) = match payload with | PStr [] -> Some UnrecognizedPayload | PStr ({pstr_desc = Pstr_eval (expr, _)} :: _) -> expr |> from_expr - | PStr ({pstr_desc = Pstr_extension _} :: _) -> Some UnrecognizedPayload + | PStr ({pstr_desc = Pstr_extension _ | Pstr_raw _} :: _) -> + Some UnrecognizedPayload | PStr ({pstr_desc = Pstr_value _} :: _) -> Some UnrecognizedPayload | PStr ({pstr_desc = Pstr_primitive _} :: _) -> Some UnrecognizedPayload | PStr ({pstr_desc = Pstr_type _} :: _) -> Some UnrecognizedPayload diff --git a/analysis/src/completion_front_end.ml b/analysis/src/completion_front_end.ml index dee20d26c9..7cd50d5a5e 100644 --- a/analysis/src/completion_front_end.ml +++ b/analysis/src/completion_front_end.ml @@ -210,8 +210,7 @@ let rec expr_to_context_path_inner ~(in_jsx_context : bool) (e : Parsetree.expression) = match e.pexp_desc with | Pexp_braces {expr} -> expr_to_context_path_inner ~in_jsx_context expr - | Pexp_constant (Pconst_string _ | Pconst_raw_source _) -> - Some Completable.CPString + | Pexp_constant (Pconst_string _) -> Some Completable.CPString | Pexp_template _ -> Some Completable.CPString | Pexp_tagged_template {tag} -> ( match expr_to_context_path ~in_jsx_context tag with @@ -1289,6 +1288,8 @@ let completion_with_parser1 ~debug ~offset ~pos_cursor ~kind_file | Pexp_object_literal fields -> List.iter (fun (_, e) -> iterator.expr iterator e) fields | Pexp_extension ({txt}, _) -> set_result (CextensionNode txt) + | Pexp_raw {kind} -> + set_result (CextensionNode (Ast_raw.extension_name kind)) | Pexp_constant _ -> set_result Cnone | Pexp_ident lid -> let lid_path = flatten_lid_check_dot lid in diff --git a/analysis/src/document_symbol.ml b/analysis/src/document_symbol.ml index ed96a799f0..92d77ccda0 100644 --- a/analysis/src/document_symbol.ml +++ b/analysis/src/document_symbol.ml @@ -20,8 +20,7 @@ let get_symbols ~source ~kind_file = | Pexp_fun _ -> Lsp.Types.SymbolKind.Function | Pexp_braces {expr} -> expr_kind expr | Pexp_constraint (e, _) -> expr_kind e - | Pexp_constant (Pconst_string _ | Pconst_raw_source _) -> - Lsp.Types.SymbolKind.String + | Pexp_constant (Pconst_string _) -> Lsp.Types.SymbolKind.String | Pexp_template _ -> Lsp.Types.SymbolKind.String | Pexp_constant (Pconst_float _ | Pconst_integer _) -> Lsp.Types.SymbolKind.Number diff --git a/analysis/src/dump_ast.ml b/analysis/src/dump_ast.ml index 53437c830e..9da9853f2c 100644 --- a/analysis/src/dump_ast.ml +++ b/analysis/src/dump_ast.ml @@ -51,7 +51,6 @@ let print_constant const = ^ ", semantic=" ^ String_literal.string_semantic payload ^ ")" - | Pconst_raw_source source -> "Pconst_raw_source(" ^ source ^ ")" | Pconst_float (s, _) -> "Pconst_float(" ^ s ^ ")" let print_core_type typ ~pos = diff --git a/analysis/src/utils.ml b/analysis/src/utils.ml index 6873491857..001350dd67 100644 --- a/analysis/src/utils.ml +++ b/analysis/src/utils.ml @@ -117,6 +117,7 @@ let identify_pexp pexp = | Pexp_assert _ -> "Pexp_assert" | Pexp_pack _ -> "Pexp_pack" | Pexp_extension _ -> "Pexp_extension" + | Pexp_raw _ -> "Pexp_raw" | Pexp_open _ -> "Pexp_open" | Pexp_await _ -> "Pexp_await" | Pexp_jsx_element _ -> "Pexp_jsx_element" diff --git a/changes/raw-extension-nodes.internal.md b/changes/raw-extension-nodes.internal.md new file mode 100644 index 0000000000..4162d933e1 --- /dev/null +++ b/changes/raw-extension-nodes.internal.md @@ -0,0 +1 @@ +- Represent `%raw`, `%ffi` and `%%raw` with dedicated parsetree nodes. https://github.com/rescript-lang/rescript/pull/8879 diff --git a/changes/raw-template-ppx.fix.md b/changes/raw-template-ppx.fix.md new file mode 100644 index 0000000000..1b72675f4f --- /dev/null +++ b/changes/raw-template-ppx.fix.md @@ -0,0 +1 @@ +- Fix template and braced payloads of `%raw` and `%ffi` compiling differently when a PPX is used. https://github.com/rescript-lang/rescript/pull/8879 diff --git a/compiler/core/js_implementation.ml b/compiler/core/js_implementation.ml index 2b1719a276..da0b5d784e 100644 --- a/compiler/core/js_implementation.ml +++ b/compiler/core/js_implementation.ml @@ -87,7 +87,8 @@ let all_module_alias (ast : Parsetree.structure) = | Pstr_attribute _ -> true | Pstr_eval _ | Pstr_value _ | Pstr_primitive _ | Pstr_type _ | Pstr_typext _ | Pstr_exception _ | Pstr_module _ | Pstr_recmodule _ - | Pstr_modtype _ | Pstr_open _ | Pstr_include _ | Pstr_extension _ -> + | Pstr_modtype _ | Pstr_open _ | Pstr_include _ | Pstr_extension _ + | Pstr_raw _ -> false) let no_export (rest : Parsetree.structure) : Parsetree.structure = diff --git a/compiler/ext/config.ml b/compiler/ext/config.ml index 863cd6353e..8b9658a611 100644 --- a/compiler/ext/config.ml +++ b/compiler/ext/config.ml @@ -1,10 +1,10 @@ -let cmi_magic_number = "Caml1999I037" +let cmi_magic_number = "Caml1999I038" (* Magic numbers for marshaled values of the *current* parsetree, whose layout changes across compiler versions. *) -and ast_impl_magic_number = "ResImpl01316" +and ast_impl_magic_number = "ResImpl01317" -and ast_intf_magic_number = "ResIntf01316" +and ast_intf_magic_number = "ResIntf01317" (* Magic numbers of the frozen Parsetree0 (OCaml 4.06) layout used on the external-PPX wire. They must never be written in front of a @@ -13,7 +13,7 @@ and ast0_impl_magic_number = "Caml1999M022" and ast0_intf_magic_number = "Caml1999N022" -and cmt_magic_number = "Caml1999T044" +and cmt_magic_number = "Caml1999T045" let get_load_path () = (Compiler_request_state.current ()).load_path let set_load_path path = (Compiler_request_state.current ()).load_path <- path diff --git a/compiler/frontend/ast_exp_handle_external.ml b/compiler/frontend/ast_exp_handle_external.ml index e110d00a4c..c2d80ab2e8 100644 --- a/compiler/frontend/ast_exp_handle_external.ml +++ b/compiler/frontend/ast_exp_handle_external.ml @@ -34,6 +34,60 @@ let handle_debugger loc (payload : Ast_payload.t) = | _ -> Location.raise_errorf ~loc "%%debugger extension doesn't accept arguments" +let raw_type () = + Ast_helper.Typ.arrow + [Parg_type {attrs = []; lbl = Nolabel; typ = Ast_helper.Typ.any ()}] + (Ast_helper.Typ.any ()) + +(* [exp] is the string constant holding the JavaScript source. *) +let raw_expression loc ~is_function (exp : Parsetree.expression) = + { + exp with + pexp_desc = + Ast_external_mk.local_external_apply loc + ~pval_prim:(Prim_name "#raw_expr") ~pval_type:(raw_type ()) [exp]; + pexp_attributes = + (match !is_function with + | None -> exp.pexp_attributes + | Some _ -> Ast_attributes.internal_expansive :: exp.pexp_attributes); + } + +let ffi_error loc = + Location.raise_errorf ~loc + "%%ffi extension can only be applied to a string containing a JavaScript \ + function such as \"(x) => ...\"" + +let ffi_expression loc ~is_function (exp : Parsetree.expression) = + (* Wrap a type constraint based on arity. + E.g. for arity 2 constrain to type (_, _) => _ *) + let wrap_type_constraint (e : Parsetree.expression) = + let loc = e.pexp_loc in + let any = Ast_helper.Typ.any ~loc:e.pexp_loc () in + let arrow ~arity = + let effective_arity = if arity = 0 then 1 else arity in + let args = + Ext_list.init effective_arity (fun _ -> + Parsetree.Parg_type {attrs = []; lbl = Nolabel; typ = any}) + in + Ast_helper.Typ.arrow ~loc args any + in + match !is_function with + | Some arity -> Ast_helper.Exp.constraint_ ~loc e (arrow ~arity) + | _ -> ffi_error loc + in + match !is_function with + | None -> ffi_error loc + | Some _ -> wrap_type_constraint (raw_expression loc ~is_function exp) + +let raw_structure loc (exp : Parsetree.expression) = + Ast_helper.Str.eval + { + exp with + pexp_desc = + Ast_external_mk.local_external_apply loc + ~pval_prim:(Prim_name "#raw_stmt") ~pval_type:(raw_type ()) [exp]; + } + let handle_raw loc payload = let is_function = ref None in match @@ -41,93 +95,40 @@ let handle_raw loc payload = with | None -> Location.raise_errorf ~loc "%%raw extension can only be applied to a string" - | Some exp -> - { - exp with - pexp_desc = - Ast_external_mk.local_external_apply loc - ~pval_prim:(Prim_name "#raw_expr") - ~pval_type: - (Ast_helper.Typ.arrow - [ - Parg_type - {attrs = []; lbl = Nolabel; typ = Ast_helper.Typ.any ()}; - ] - (Ast_helper.Typ.any ())) - [exp]; - pexp_attributes = - (match !is_function with - | None -> exp.pexp_attributes - | Some _ -> Ast_attributes.internal_expansive :: exp.pexp_attributes); - } + | Some exp -> raw_expression loc ~is_function exp let handle_ffi ~loc ~payload = let is_function = ref None in - let err () = - Location.raise_errorf ~loc - "%%ffi extension can only be applied to a string containing a JavaScript \ - function such as \"(x) => ...\"" - in match Ast_payload.raw_as_string_exp_exn ~kind:Raw_exp ~is_function payload with - | None -> err () - | Some exp -> - (* Wrap a type constraint based on arity. - E.g. for arity 2 constrain to type (_, _) => _ *) - let wrap_type_constraint (e : Parsetree.expression) = - let loc = e.pexp_loc in - let any = Ast_helper.Typ.any ~loc:e.pexp_loc () in - let arrow ~arity = - let effective_arity = if arity = 0 then 1 else arity in - let args = - Ext_list.init effective_arity (fun _ -> - Parsetree.Parg_type {attrs = []; lbl = Nolabel; typ = any}) - in - Ast_helper.Typ.arrow ~loc args any - in - match !is_function with - | Some arity -> Ast_helper.Exp.constraint_ ~loc e (arrow ~arity) - | _ -> err () - in - wrap_type_constraint - { - exp with - pexp_desc = - Ast_external_mk.local_external_apply loc - ~pval_prim:(Prim_name "#raw_expr") - ~pval_type: - (Ast_helper.Typ.arrow - [ - Parg_type - {attrs = []; lbl = Nolabel; typ = Ast_helper.Typ.any ()}; - ] - (Ast_helper.Typ.any ())) - [exp]; - pexp_attributes = - (match !is_function with - | None -> exp.pexp_attributes - | Some _ -> Ast_attributes.internal_expansive :: exp.pexp_attributes); - } + | None -> ffi_error loc + | Some exp -> ffi_expression loc ~is_function exp let handle_raw_structure loc payload = match Ast_payload.raw_as_string_exp_exn ~kind:Raw_program payload with - | Some exp -> - Ast_helper.Str.eval - { - exp with - pexp_desc = - Ast_external_mk.local_external_apply loc - ~pval_prim:(Prim_name "#raw_stmt") - ~pval_type: - (Ast_helper.Typ.arrow - [ - Parg_type - {attrs = []; lbl = Nolabel; typ = Ast_helper.Typ.any ()}; - ] - (Ast_helper.Typ.any ())) - [exp]; - } + | Some exp -> raw_structure loc exp | None -> Location.raise_errorf ~loc "%%%%raw extension can only be applied to a string" + +(* The string constant holding the JavaScript source of a raw node, validated + like a raw payload. *) +let raw_source_expression ~(kind : Js_raw_info.raw_kind) ?is_function + (source : Parsetree.raw_source) = + let str = Ast_raw.js_source source in + let loc = Ast_raw.payload_loc source in + Ast_payload.validate_raw_source ~kind ?is_function ~loc + ~offset:(Bs_flow_ast_utils.flow_deli_offset (Some "js")) + str; + Ast_helper.Exp.constant ~loc (Ast_helper.Const.string str) + +let handle_raw_node ~loc (kind : Parsetree.raw_kind) source = + let is_function = ref None in + let exp = raw_source_expression ~kind:Raw_exp ~is_function source in + match kind with + | Raw_expression -> raw_expression loc ~is_function exp + | Raw_function -> ffi_expression loc ~is_function exp + +let handle_raw_structure_node ~loc source = + raw_structure loc (raw_source_expression ~kind:Raw_program source) diff --git a/compiler/frontend/ast_exp_handle_external.mli b/compiler/frontend/ast_exp_handle_external.mli index c8cb803bee..f65304286e 100644 --- a/compiler/frontend/ast_exp_handle_external.mli +++ b/compiler/frontend/ast_exp_handle_external.mli @@ -30,3 +30,14 @@ val handle_raw : Location.t -> Ast_payload.t -> Parsetree.expression val handle_raw_structure : Location.t -> Ast_payload.t -> Parsetree.structure_item + +val handle_raw_node : + loc:Location.t -> + Parsetree.raw_kind -> + Parsetree.raw_source -> + Parsetree.expression +(** Lower [%raw] or [%ffi], located at [loc]. *) + +val handle_raw_structure_node : + loc:Location.t -> Parsetree.raw_source -> Parsetree.structure_item +(** Lower [%%raw], located at [loc]. *) diff --git a/compiler/frontend/bs_builtin_ppx.ml b/compiler/frontend/bs_builtin_ppx.ml index de147ec53c..13cbcf9cd5 100644 --- a/compiler/frontend/bs_builtin_ppx.ml +++ b/compiler/frontend/bs_builtin_ppx.ml @@ -143,7 +143,7 @@ let expr_mapper ~async_context ~in_function_def (self : mapper) Parg_type {attrs = []; lbl = Nolabel; typ = Ast_helper.Typ.any ()}; ] (Ast_helper.Typ.any ())) - [Ast_helper.Exp.constant ~loc (Pconst_raw_source source)] + [Ast_helper.Exp.constant ~loc (Ast_helper.Const.string source)] in Ast_helper.Exp.constraint_ ~loc ~attrs:(self.attributes self e.pexp_attributes) @@ -151,6 +151,8 @@ let expr_mapper ~async_context ~in_function_def (self : mapper) (Ast_comb.to_regexp_type loc) | Pexp_extension extension -> Ast_exp_extension.handle_extension e self extension + | Pexp_raw {kind; name_loc; source} -> + Ast_exp_handle_external.handle_raw_node ~loc:name_loc kind source | Pexp_constant (Pconst_integer (s, Some 'l')) -> {e with pexp_desc = Pexp_constant (Pconst_integer (s, None))} (* End rewriting *) @@ -752,6 +754,9 @@ let rec structure_mapper ~await_context (self : mapper) (stru : Ast_structure.t) | Pstr_extension (({txt = "raw"; loc}, payload), _attrs) -> Ast_exp_handle_external.handle_raw_structure loc payload :: structure_mapper ~await_context self rest + | Pstr_raw {name_loc; source} -> + Ast_exp_handle_external.handle_raw_structure_node ~loc:name_loc source + :: structure_mapper ~await_context self rest (* | Pstr_extension (({txt = "i"}, _),_) -> structure_mapper self rest *) diff --git a/compiler/gentype/annotation.ml b/compiler/gentype/annotation.ml index eed78411a2..d795a752b3 100644 --- a/compiler/gentype/annotation.ml +++ b/compiler/gentype/annotation.ml @@ -73,7 +73,8 @@ let rec get_attribute_payload check_text (attributes : Typedtree.attributes) = match payload with | PStr [] -> Some UnrecognizedPayload | PStr ({pstr_desc = Pstr_eval (expr, _)} :: _) -> expr |> from_expr - | PStr ({pstr_desc = Pstr_extension _} :: _) -> Some UnrecognizedPayload + | PStr ({pstr_desc = Pstr_extension _ | Pstr_raw _} :: _) -> + Some UnrecognizedPayload | PStr ({pstr_desc = Pstr_value _} :: _) -> Some UnrecognizedPayload | PStr ({pstr_desc = Pstr_primitive _} :: _) -> Some UnrecognizedPayload | PStr ({pstr_desc = Pstr_type _} :: _) -> Some UnrecognizedPayload diff --git a/compiler/ml/ast_helper.ml b/compiler/ml/ast_helper.ml index 8931123e97..2ea399bf5d 100644 --- a/compiler/ml/ast_helper.ml +++ b/compiler/ml/ast_helper.ml @@ -194,6 +194,8 @@ module Exp = struct let pack ?loc ?attrs a = mk ?loc ?attrs (Pexp_pack a) let open_ ?loc ?attrs a b c = mk ?loc ?attrs (Pexp_open (a, b, c)) let extension ?loc ?attrs a = mk ?loc ?attrs (Pexp_extension a) + let raw ?loc ?attrs ~name_loc kind source = + mk ?loc ?attrs (Pexp_raw {kind; name_loc; source}) let regexp ?loc ?attrs pattern flags = mk ?loc ?attrs (Pexp_regexp {pattern; flags}) let template ?loc ?attrs source_segments values = @@ -318,6 +320,8 @@ module Str = struct let open_ ?loc a = mk ?loc (Pstr_open a) let include_ ?loc a = mk ?loc (Pstr_include a) let extension ?loc ?(attrs = []) a = mk ?loc (Pstr_extension (a, attrs)) + let raw ?loc ?(attrs = []) ~name_loc source = + mk ?loc (Pstr_raw {name_loc; source; attrs}) let attribute ?loc a = mk ?loc (Pstr_attribute a) end diff --git a/compiler/ml/ast_helper.mli b/compiler/ml/ast_helper.mli index 19d502945e..e840250f02 100644 --- a/compiler/ml/ast_helper.mli +++ b/compiler/ml/ast_helper.mli @@ -238,6 +238,13 @@ module Exp : sig val open_ : ?loc:loc -> ?attrs:attrs -> override_flag -> lid -> expression -> expression val extension : ?loc:loc -> ?attrs:attrs -> extension -> expression + val raw : + ?loc:loc -> + ?attrs:attrs -> + name_loc:loc -> + raw_kind -> + raw_source -> + expression val tagged_template : ?loc:loc -> ?attrs:attrs -> @@ -439,6 +446,8 @@ module Str : sig val open_ : ?loc:loc -> open_description -> structure_item val include_ : ?loc:loc -> include_declaration -> structure_item val extension : ?loc:loc -> ?attrs:attrs -> extension -> structure_item + val raw : + ?loc:loc -> ?attrs:attrs -> name_loc:loc -> raw_source -> structure_item val attribute : ?loc:loc -> attribute -> structure_item end diff --git a/compiler/ml/ast_iterator.ml b/compiler/ml/ast_iterator.ml index a8dfb9d199..29617acb1e 100644 --- a/compiler/ml/ast_iterator.ml +++ b/compiler/ml/ast_iterator.ml @@ -278,6 +278,10 @@ module M = struct | Pstr_extension (x, attrs) -> sub.extension sub x; sub.attributes sub attrs + | Pstr_raw {name_loc; source; attrs} -> + sub.location sub name_loc; + sub.location sub (Ast_raw.source_loc source); + sub.attributes sub attrs | Pstr_attribute x -> sub.attribute sub x end @@ -416,6 +420,9 @@ module E = struct iter_loc sub lid; sub.expr sub e | Pexp_extension x -> sub.extension sub x + | Pexp_raw {name_loc; source} -> + sub.location sub name_loc; + sub.location sub (Ast_raw.source_loc source) | Pexp_regexp _ -> () | Pexp_template {values} -> List.iter (sub.expr sub) values | Pexp_tagged_template {tag; values} -> diff --git a/compiler/ml/ast_mapper.ml b/compiler/ml/ast_mapper.ml index ad37865225..b3652f28fa 100644 --- a/compiler/ml/ast_mapper.ml +++ b/compiler/ml/ast_mapper.ml @@ -71,6 +71,10 @@ let map_opt f = function let map_loc sub {loc; txt} = {loc = sub.location sub loc; txt} +let map_raw_source sub : raw_source -> raw_source = function + | Raw_string source -> Raw_string (map_loc sub source) + | Raw_template source -> Raw_template (map_loc sub source) + module T = struct (* Type expressions for the core language *) @@ -287,6 +291,11 @@ module M = struct | Pstr_include x -> include_ ~loc (sub.include_declaration sub x) | Pstr_extension (x, attrs) -> extension ~loc (sub.extension sub x) ~attrs:(sub.attributes sub attrs) + | Pstr_raw {name_loc; source; attrs} -> + raw ~loc + ~name_loc:(sub.location sub name_loc) + (map_raw_source sub source) + ~attrs:(sub.attributes sub attrs) | Pstr_attribute x -> attribute ~loc (sub.attribute sub x) end @@ -418,6 +427,11 @@ module E = struct | Pexp_open (ovf, lid, e) -> open_ ~loc ~attrs ovf (map_loc sub lid) (sub.expr sub e) | Pexp_extension x -> extension ~loc ~attrs (sub.extension sub x) + | Pexp_raw {kind; name_loc; source} -> + raw ~loc ~attrs + ~name_loc:(sub.location sub name_loc) + kind + (map_raw_source sub source) | Pexp_await e -> await ~loc ~attrs (sub.expr sub e) | Pexp_jsx_element (Jsx_fragment diff --git a/compiler/ml/ast_mapper_from0.ml b/compiler/ml/ast_mapper_from0.ml index 3ad2d426ce..4fc85334ac 100644 --- a/compiler/ml/ast_mapper_from0.ml +++ b/compiler/ml/ast_mapper_from0.ml @@ -135,7 +135,7 @@ let map_constant ~loc = function Pconst_char {source = String_literal.encode_char_source semantic; semantic} | Pconst_string (s, Some ("js" | "*j")) -> source_string ~loc s | Pconst_string (s, None) -> semantic_string s - | Pconst_string (s, Some "json") -> Pconst_raw_source s + | Pconst_string (s, Some "json") -> source_string ~loc s (* Other v0 quotation delimiters are syntax, not part of the string value. Tagged ReScript templates are represented as applications before PPX. *) | Pconst_string (semantic, Some _) -> semantic_string semantic @@ -151,6 +151,7 @@ let is_raw_source_extension = function | "raw" | "ffi" -> true | _ -> false +(* A braced string or a template is mapped as usual. *) let map_raw_source_payload sub = function | PStr [ @@ -165,12 +166,22 @@ let map_raw_source_payload sub = function eval_attributes ); pstr_loc; }; - ] -> + ] + when not + (has_template_attr pexp_attributes + || List.exists (fun ({txt}, _) -> txt = "res.braces") pexp_attributes + ) -> + (* The payload is JavaScript source: keep it as the string's spelling. *) + let payload = + match String_literal.string_from_source s with + | Some payload -> payload + | None -> String_literal.string_from_semantic s + in let expression = Ast_helper.Exp.constant ~loc:(sub.location sub pexp_loc) ~attrs:(sub.attributes sub pexp_attributes) - (Pt.Pconst_raw_source s) + (Pt.Pconst_string payload) in Some (Pt.PStr @@ -182,6 +193,30 @@ let map_raw_source_payload sub = function ]) | _ -> None +(* The source of a plain v0 [%raw], [%ffi] or [%%raw] payload: a string, or a + template marked as such. Other payloads stay extensions. *) +let raw_source_from0 sub = function + | PStr + [ + { + pstr_desc = + Pstr_eval + ( { + pexp_desc = Pexp_constant (Pconst_string (s, delimiter)); + pexp_loc; + pexp_attributes; + }, + [] ); + }; + ] -> ( + let loc = sub.location sub pexp_loc in + match pexp_attributes with + | [] -> Some (Pt.Raw_string {txt = s; loc}) + | attrs when has_template_attr attrs && remove_template_attr attrs = [] -> + Some (Pt.Raw_template {txt = template_source_from0 (s, delimiter); loc}) + | _ -> None) + | _ -> None + let for_of_attr_name = "_res.for_of" let for_await_of_attr_name = "_res.for_await_of" @@ -541,6 +576,14 @@ module M = struct | Pstr_class_type () -> failwith "Pstr_class_type is no longer present in ReScript" | Pstr_include x -> include_ ~loc (sub.include_declaration sub x) + | Pstr_extension ((({txt = "raw"} as name), payload), attrs) + when raw_source_from0 sub payload <> None -> ( + match raw_source_from0 sub payload with + | Some source -> + raw ~loc + ~name_loc:(sub.location sub name.loc) + source ~attrs:(sub.attributes sub attrs) + | None -> assert false) | Pstr_extension (x, attrs) -> extension ~loc (sub.extension sub x) ~attrs:(sub.attributes sub attrs) | Pstr_attribute x -> attribute ~loc (sub.attribute sub x) @@ -1338,7 +1381,11 @@ module E = struct pattern flags | _ -> malformed ~loc:(sub.location sub source_loc)) | _ -> malformed ~loc) - | Pexp_extension x -> extension ~loc ~attrs (sub.extension sub x) + | Pexp_extension (({txt = name; loc = name_loc}, payload) as x) -> ( + match (Ast_raw.kind_of_name name, raw_source_from0 sub payload) with + | Some kind, Some source -> + raw ~loc ~attrs ~name_loc:(sub.location sub name_loc) kind source + | _ -> extension ~loc ~attrs (sub.extension sub x)) | Pexp_unreachable -> assert false end diff --git a/compiler/ml/ast_mapper_to0.ml b/compiler/ml/ast_mapper_to0.ml index e5e8112580..d469b0e0cf 100644 --- a/compiler/ml/ast_mapper_to0.ml +++ b/compiler/ml/ast_mapper_to0.ml @@ -80,11 +80,31 @@ let map_constant = function | Pconst_char {semantic} -> Pconst_char semantic | Pconst_string payload -> Pconst_string (String_literal.string_source payload, Some "js") - | Pconst_raw_source s -> Pconst_string (s, Some "js") | Pconst_float (s, suffix) -> Pconst_float (s, suffix) let template_attr = (Location.mknoloc "res.template", Pt.PStr []) +(* v0 spells raw JavaScript as the extension it is written as, with the source + in a ["js"]-delimited string. *) +let raw_extension sub ~name ~name_loc (source : Parsetree.raw_source) : + Pt.extension = + let constant ~attrs ({txt; loc} : string Location.loc) = + Ast_helper0.Exp.constant ~loc:(sub.location sub loc) ~attrs + (Pt.Pconst_string (txt, Some "js")) + in + let expression = + match source with + | Raw_string source -> constant ~attrs:[] source + | Raw_template source -> constant ~attrs:[template_attr] source + in + ( {txt = name; loc = sub.location sub name_loc}, + Pt.PStr + [ + Ast_helper0.Str.eval + ~loc:(sub.location sub (Ast_raw.payload_loc source)) + expression; + ] ) + let add_template_attr attrs = if Ext_list.exists attrs (fun ({txt}, _) -> txt = "res.template") then attrs else template_attr :: attrs @@ -410,6 +430,10 @@ module M = struct | Pstr_include x -> include_ ~loc (sub.include_declaration sub x) | Pstr_extension (x, attrs) -> extension ~loc (sub.extension sub x) ~attrs:(sub.attributes sub attrs) + | Pstr_raw {name_loc; source; attrs} -> + extension ~loc + (raw_extension sub ~name:"raw" ~name_loc source) + ~attrs:(sub.attributes sub attrs) | Pstr_attribute x -> attribute ~loc (sub.attribute sub x) end @@ -810,6 +834,9 @@ module E = struct | Pexp_open (ovf, lid, e) -> open_ ~loc ~attrs ovf (map_loc sub lid) (sub.expr sub e) | Pexp_extension x -> extension ~loc ~attrs (sub.extension sub x) + | Pexp_raw {kind; name_loc; source} -> + extension ~loc ~attrs + (raw_extension sub ~name:(Ast_raw.extension_name kind) ~name_loc source) | Pexp_await e -> (* Single v0 attribute slot for two nodes: the await node's own attributes go in front of the [res.await] marker, the inner diff --git a/compiler/ml/ast_payload.ml b/compiler/ml/ast_payload.ml index 6f5295d575..7dcfdc3475 100644 --- a/compiler/ml/ast_payload.ml +++ b/compiler/ml/ast_payload.ml @@ -196,6 +196,11 @@ let validate_raw_source ~(kind : Js_raw_info.raw_kind) ?is_function ~loc ~offset let raw_as_string_exp_exn ~(kind : Js_raw_info.raw_kind) ?is_function (x : t) : Parsetree.expression option = + let braced = + match x with + | PStr [{pstr_desc = Pstr_eval ({pexp_desc = Pexp_braces _}, _)}] -> true + | _ -> false + in let string_expression = match unwrap_payload_expression x with (* TODO also need detect empty phrase case *) @@ -223,17 +228,30 @@ let raw_as_string_exp_exn ~(kind : Js_raw_info.raw_kind) ?is_function (x : t) : }; ] -> ( match constant with - | Pconst_raw_source source -> - Some (source, Bs_flow_ast_utils.flow_deli_offset (Some "js"), expression) - | Pconst_string payload -> + | Pconst_string payload when braced -> + (* A braced payload is an ordinary string expression, so its escapes + are decoded. *) Some (String_literal.string_semantic payload, 0, expression) + | Pconst_string payload -> + (* A raw payload is JavaScript source, so the string is taken as + written. *) + Some + ( String_literal.string_source payload, + Bs_flow_ast_utils.flow_deli_offset (Some "js"), + expression ) | _ -> None) | _ -> None in match string_expression with | Some (str, offset, ({pexp_loc = loc} as expression)) -> validate_raw_source ~kind ?is_function ~loc ~offset str; - Some {expression with pexp_desc = Pexp_constant (Pconst_raw_source str)} + Some + { + expression with + pexp_desc = + Pexp_constant + (Pconst_string (String_literal.string_from_semantic str)); + } | None -> None type lid = string Asttypes.loc diff --git a/compiler/ml/ast_raw.ml b/compiler/ml/ast_raw.ml new file mode 100644 index 0000000000..8bebc880dd --- /dev/null +++ b/compiler/ml/ast_raw.ml @@ -0,0 +1,82 @@ +open Parsetree + +let extension_name = function + | Raw_expression -> "raw" + | Raw_function -> "ffi" + +let source_loc = function + | Raw_string {loc} | Raw_template {loc} -> loc + +let js_source = function + | Raw_string {txt} -> txt + | Raw_template {txt} -> String_literal.decode_raw_template_source txt + +let payload_loc = function + | Raw_string {loc} -> loc + | Raw_template {loc} -> + (* The template source ends before the closing backquote. *) + if loc.loc_ghost then loc + else + { + loc with + loc_end = {loc.loc_end with pos_cnum = loc.loc_end.pos_cnum + 1}; + } + +let kind_of_name = function + | "raw" -> Some Raw_expression + | "ffi" -> Some Raw_function + | _ -> None + +let of_payload : payload -> raw_source option = function + | PStr + [ + {pstr_desc = Pstr_eval ({pexp_desc; pexp_loc; pexp_attributes = []}, [])}; + ] -> ( + match pexp_desc with + | Pexp_constant (Pconst_string payload) -> + Some + (Raw_string {txt = String_literal.string_source payload; loc = pexp_loc}) + | Pexp_template {source_segments = [source]; values = []} -> + Some (Raw_template source) + | _ -> None) + | _ -> None + +(* Whether [source] can be written between double quotes as it is: it has no + unescaped double quote. *) +let fits_double_quotes source = + let rec loop index = + if index >= String.length source then true + else + match String.unsafe_get source index with + | '"' -> false + | '\\' -> index + 1 < String.length source && loop (index + 2) + | _ -> loop (index + 1) + in + loop 0 + +let to_extension ~name ~name_loc source : extension = + let loc = payload_loc source in + let expression = + match source with + | Raw_string source -> ( + match + if fits_double_quotes source.txt then + String_literal.string_from_source source.txt + else None + with + | Some payload -> Ast_helper.Exp.constant ~loc (Pconst_string payload) + | None -> + Ast_helper.Exp.template ~loc + [ + { + source with + txt = String_literal.encode_raw_template_source source.txt; + }; + ] + []) + | Raw_template source -> Ast_helper.Exp.template ~loc [source] [] + in + ({txt = name; loc = name_loc}, PStr [Ast_helper.Str.eval ~loc expression]) + +let uninterpreted ~loc name = + Location.errorf ~loc "Uninterpreted extension '%s'." name diff --git a/compiler/ml/ast_raw.mli b/compiler/ml/ast_raw.mli new file mode 100644 index 0000000000..1fabb6d7c3 --- /dev/null +++ b/compiler/ml/ast_raw.mli @@ -0,0 +1,30 @@ +(** [%raw], [%ffi] and [%%raw]. *) + +val extension_name : Parsetree.raw_kind -> string +(** ["raw"] or ["ffi"]. *) + +val source_loc : Parsetree.raw_source -> Location.t + +val js_source : Parsetree.raw_source -> string +(** The JavaScript source that a raw node stands for. *) + +val payload_loc : Parsetree.raw_source -> Location.t +(** The location of the string or template literal holding the source. *) + +val kind_of_name : string -> Parsetree.raw_kind option +(** The kind of [%raw] or [%ffi]. *) + +val of_payload : Parsetree.payload -> Parsetree.raw_source option +(** The source in a parsed [%raw], [%ffi] or [%%raw] payload, if it is a plain + string or template. *) + +val to_extension : + name:string -> + name_loc:Location.t -> + Parsetree.raw_source -> + Parsetree.extension +(** The extension as parsed, for passes that treat raw JavaScript like the + extension it is written as. *) + +val uninterpreted : loc:Location.t -> string -> Location.error +(** The error for an extension that no compiler pass interpreted. *) diff --git a/compiler/ml/depend.ml b/compiler/ml/depend.ml index a2e2240685..82960f38b3 100644 --- a/compiler/ml/depend.ml +++ b/compiler/ml/depend.ml @@ -320,6 +320,7 @@ let rec add_expr bv exp = | Pstr_eval ({pexp_desc = Pexp_construct (c, {txt = []})}, _) -> add bv c | _ -> handle_extension e) | Pexp_extension e -> handle_extension e + | Pexp_raw _ -> () | Pexp_await e -> add_expr bv e | Pexp_jsx_element (Jsx_fragment {jsx_fragment_children = children}) -> add_jsx_children bv children @@ -539,7 +540,7 @@ and add_struct_item (bv, m) item : _ String_map.t * _ String_map.t = add_names s; let add = String_map.fold String_map.add m' in (add bv, add m) - | Pstr_attribute _ -> (bv, m) + | Pstr_attribute _ | Pstr_raw _ -> (bv, m) | Pstr_extension (e, _) -> handle_extension e; (bv, m) diff --git a/compiler/ml/parsetree.ml b/compiler/ml/parsetree.ml index bd865de669..f9a008d7b8 100644 --- a/compiler/ml/parsetree.ml +++ b/compiler/ml/parsetree.ml @@ -47,11 +47,6 @@ type constant = legacy three-digit decimal escapes to hexadecimal escapes. Compiler-created Payloads are constructed through [String_literal], which validates this relationship. *) - | Pconst_raw_source of string - (* JavaScript source carried by a compiler extension such as [raw] or [ffi]. - For example, [%raw("x + 1")] stores ["x + 1"]. The extension - interprets the string as JavaScript source rather than as a ReScript - runtime string value. *) | Pconst_float of string * char option (* 3.4 2e5 1.4e-4 @@ -401,6 +396,9 @@ and expression_desc = let! open M in E *) | Pexp_extension of extension (* [%id] *) + | Pexp_raw of {kind: raw_kind; name_loc: Location.t; source: raw_source} + (* %raw("...") and %ffi("..."): JavaScript source. [name_loc] is the + location of [%raw] or [%ffi]. *) (* . *) | Pexp_await of expression | Pexp_jsx_element of jsx_element @@ -809,7 +807,22 @@ and structure_item_desc = | Pstr_include of include_declaration (* include ME *) | Pstr_attribute of attribute (* [@@@id] *) | Pstr_extension of extension * attributes -(* [%%id] *) + (* [%%id] *) + | Pstr_raw of {name_loc: Location.t; source: raw_source; attrs: attributes} +(* %%raw("..."): JavaScript statements. [name_loc] is the location of + [%%raw]. *) + +and raw_kind = + | Raw_expression (* %raw: any JavaScript expression *) + | Raw_function (* %ffi: a JavaScript function *) + +and raw_source = + | Raw_string of string loc + (* %raw("..."): the JavaScript source, which is also the text between the + quotes. The location is the string literal's. *) + | Raw_template of string loc +(* %raw(`...`): the template source as written, from which + [String_literal.decode_raw_template_source] gives the JavaScript source *) and value_constraint = { pvc_newtypes: string loc list; diff --git a/compiler/ml/pprintast.ml b/compiler/ml/pprintast.ml index 00bb5711e1..11d9bbd5e9 100644 --- a/compiler/ml/pprintast.ml +++ b/compiler/ml/pprintast.ml @@ -252,8 +252,6 @@ let constant f = function | Pconst_string payload -> pp f "{js|%a|js}" print_string_with_byte_width (String_literal.string_source payload) - | Pconst_raw_source source -> - pp f "{js|%a|js}" print_string_with_byte_width source | Pconst_integer (i, None) -> paren (i.[0] = '-') (fun f -> pp f "%s") f i | Pconst_integer (i, Some m) -> paren (i.[0] = '-') (fun f (i, m) -> pp f "%s%c" i m) f (i, m) @@ -789,6 +787,11 @@ and expression ctxt f x = in pp f "@[<2>`%s@;%a@]" l (simple_expr ctxt) payload | Pexp_extension e -> extension ctxt f e + | Pexp_raw {kind; name_loc; source} -> + extension ctxt f + (Ast_raw.to_extension + ~name:(Ast_raw.extension_name kind) + ~name_loc source) | Pexp_await e -> pp f "@[await@ %a@]" (simple_expr ctxt) e | Pexp_regexp {pattern; flags} -> pp f "/%s/%s" pattern flags | Pexp_template {source_segments; values} -> @@ -1321,6 +1324,9 @@ and structure_item ctxt f x = | Pstr_extension (e, a) -> item_extension ctxt f e; item_attributes ctxt f a + | Pstr_raw {name_loc; source; attrs} -> + item_extension ctxt f (Ast_raw.to_extension ~name:"raw" ~name_loc source); + item_attributes ctxt f attrs and type_param ctxt f (ct, a) = pp f "%s%a" (type_variance a) (core_type ctxt) ct diff --git a/compiler/ml/printast.ml b/compiler/ml/printast.ml index 574b3c7bb3..aae9592541 100644 --- a/compiler/ml/printast.ml +++ b/compiler/ml/printast.ml @@ -66,7 +66,6 @@ let fmt_constant f x = fprintf f "PConst_string (source=%S, semantic=%S)" (String_literal.string_source payload) (String_literal.string_semantic payload) - | Pconst_raw_source source -> fprintf f "PConst_raw_source %S" source | Pconst_float (s, m) -> fprintf f "PConst_float (%s,%a)" s fmt_char_option m let fmt_mutable_flag f x = @@ -140,6 +139,13 @@ let fixed_value i ppf (x : fixed_value) = | Pfixed_json source -> line i ppf "Pfixed_json %S %a\n" source.txt fmt_location source.loc +let raw_source i ppf (source : raw_source) = + match source with + | Raw_string {txt; loc} -> + line i ppf "Raw_string %S %a\n" txt fmt_location loc + | Raw_template {txt; loc} -> + line i ppf "Raw_template %S %a\n" txt fmt_location loc + let rec core_type i ppf x = line i ppf "core_type %a\n" fmt_location x.ptyp_loc; attributes i ppf x.ptyp_attributes; @@ -413,6 +419,11 @@ and expression i ppf x = | Pexp_open (ovf, m, e) -> line i ppf "Pexp_open %a \"%a\"\n" fmt_override_flag ovf fmt_longident_loc m; expression i ppf e + | Pexp_raw {kind; name_loc; source} -> + line i ppf "Pexp_raw \"%s\" %a\n" + (Ast_raw.extension_name kind) + fmt_location name_loc; + raw_source i ppf source | Pexp_extension (s, arg) -> line i ppf "Pexp_extension \"%s\"\n" s.txt; payload i ppf arg @@ -746,6 +757,10 @@ and structure_item i ppf x = line i ppf "Pstr_include"; attributes i ppf incl.pincl_attributes; module_expr i ppf incl.pincl_mod + | Pstr_raw {name_loc; source; attrs} -> + line i ppf "Pstr_raw %a\n" fmt_location name_loc; + attributes i ppf attrs; + raw_source i ppf source | Pstr_extension ((s, arg), attrs) -> line i ppf "Pstr_extension \"%s\"\n" s.txt; attributes i ppf attrs; diff --git a/compiler/ml/string_literal.ml b/compiler/ml/string_literal.ml index 88579e1a3c..ceb25cee63 100644 --- a/compiler/ml/string_literal.ml +++ b/compiler/ml/string_literal.ml @@ -252,6 +252,29 @@ let decode_raw_template_source source = loop 0; Buffer.contents buffer +(* The inverse of [decode_raw_template_source]: escape backquotes, [${] and + the backslashes that decoding would otherwise consume. *) +let encode_raw_template_source source = + let length = String.length source in + let buffer = Buffer.create length in + String.iteri + (fun index character -> + let next = + if index + 1 < length then Some (String.unsafe_get source (index + 1)) + else None + in + match character with + | '`' -> Buffer.add_string buffer "\\`" + | '$' when next = Some '{' -> Buffer.add_string buffer "\\$" + | '\\' -> ( + match next with + | None | Some ('\\' | '$' | '`' | '\n' | '\r' | '\226') -> + Buffer.add_string buffer "\\\\" + | Some _ -> Buffer.add_char buffer character) + | _ -> Buffer.add_char buffer character) + source; + Buffer.contents buffer + type encode_js_mode = String | Template let encode_js mode s = diff --git a/compiler/ml/string_literal.mli b/compiler/ml/string_literal.mli index f3dda64277..eab295699d 100644 --- a/compiler/ml/string_literal.mli +++ b/compiler/ml/string_literal.mli @@ -61,6 +61,10 @@ val decode_raw_template_source : string -> string Backslashes belonging to the embedded JavaScript, such as [\\n], are preserved. *) +val encode_raw_template_source : string -> string +(** The inverse of [decode_raw_template_source]: the template source for the + given JavaScript source. *) + val encode_js_string : string -> string (** Encode a semantic UTF-8 string as a canonical JavaScript string-literal body. *) diff --git a/compiler/ml/typecore.ml b/compiler/ml/typecore.ml index 38f4e491c1..a8766cee57 100644 --- a/compiler/ml/typecore.ml +++ b/compiler/ml/typecore.ml @@ -197,7 +197,7 @@ let iter_expression f e = f e; match e.pexp_desc with | Pexp_braces {expr = inner} -> expr inner - | Pexp_extension _ (* we don't iterate under extension point *) + | Pexp_extension _ (* we don't iterate under extension point *) | Pexp_raw _ | Pexp_ident _ | Pexp_constant _ -> () | Pexp_fun {params; body} -> @@ -288,7 +288,8 @@ let iter_expression f e = | Pstr_eval (e, _) -> expr e | Pstr_value (_, pel) -> List.iter binding pel | Pstr_primitive _ | Pstr_type _ | Pstr_typext _ | Pstr_exception _ - | Pstr_modtype _ | Pstr_open _ | Pstr_attribute _ | Pstr_extension _ -> + | Pstr_modtype _ | Pstr_open _ | Pstr_attribute _ | Pstr_extension _ + | Pstr_raw _ -> () | Pstr_include {pincl_mod = me} | Pstr_module {pmb_expr = me} -> module_expr me @@ -337,7 +338,6 @@ let constant : Parsetree.constant -> (Asttypes.constant, error) result = | Pconst_char {semantic} -> Ok (Const_char semantic) | Pconst_string payload -> Ok (Const_string (String_literal.string_semantic payload)) - | Pconst_raw_source s -> Ok (Const_string s) | Pconst_float (f, None) -> Ok (Const_float f) | Pconst_float (f, Some c) -> Error (Unknown_literal (f, c)) @@ -3699,6 +3699,11 @@ and type_expect_ ?deprecated_context ~context ?(recarg = Rejected) env sexp | _ -> raise (Error (loc, env, Invalid_extension_constructor_payload))) | Pexp_extension ext -> raise (Error_forward (Builtin_attributes.error_of_extension ext)) + | Pexp_raw {kind; name_loc} -> + (* The built-in PPX lowers raw JavaScript before type checking. *) + raise + (Error_forward + (Ast_raw.uninterpreted ~loc:name_loc (Ast_raw.extension_name kind))) | Pexp_await _ -> (* should be handled earlier *) assert false | Pexp_jsx_element _ -> raise (Error (sexp.pexp_loc, Env.empty, Jsx_not_enabled)) diff --git a/compiler/ml/typemod.ml b/compiler/ml/typemod.ml index 6a2d5a3038..00b78b3fce 100644 --- a/compiler/ml/typemod.ml +++ b/compiler/ml/typemod.ml @@ -1574,6 +1574,9 @@ and type_structure ?(toplevel = false) funct_body anchor env sstr = (Tstr_include incl, sg, new_env) | Pstr_extension (ext, _attrs) -> raise (Error_forward (Builtin_attributes.error_of_extension ext)) + | Pstr_raw {name_loc} -> + (* The built-in PPX lowers raw JavaScript before type checking. *) + raise (Error_forward (Ast_raw.uninterpreted ~loc:name_loc "raw")) | Pstr_attribute x -> Builtin_attributes.warning_attribute x; (Tstr_attribute x, [], env) diff --git a/compiler/syntax/src/res_comments_table.ml b/compiler/syntax/src/res_comments_table.ml index 72140ca738..f69fcf9689 100644 --- a/compiler/syntax/src/res_comments_table.ml +++ b/compiler/syntax/src/res_comments_table.ml @@ -524,6 +524,10 @@ and walk_structure_item si t comments = walk_module_type_declaration mod_typ_decl t comments | Pstr_attribute attribute -> walk_attribute attribute t comments | Pstr_extension (extension, _) -> walk_extension extension t comments + | Pstr_raw {name_loc; source} -> + walk_extension + (Ast_raw.to_extension ~name:"raw" ~name_loc source) + t comments | Pstr_include include_declaration -> walk_include_declaration include_declaration t comments | Pstr_exception extension_constructor -> @@ -1126,6 +1130,12 @@ and walk_expression expr t comments = ExprRecordRow (Location.mkloc (Longident.Lident s.txt) s.loc, e))) t comments | Pexp_extension extension -> walk_extension extension t comments + | Pexp_raw {kind; name_loc; source} -> + walk_extension + (Ast_raw.to_extension + ~name:(Ast_raw.extension_name kind) + ~name_loc source) + t comments | Pexp_letexception (extension_constructor, expr2) -> let leading, comments = partition_leading_trailing comments expr.pexp_loc in attach t.leading diff --git a/compiler/syntax/src/res_core.ml b/compiler/syntax/src/res_core.ml index b990394e08..222cffc834 100644 --- a/compiler/syntax/src/res_core.ml +++ b/compiler/syntax/src/res_core.ml @@ -2100,10 +2100,14 @@ and parse_atomic_expr p = | Lbracket -> parse_array_exp p | Lbrace -> parse_braced_or_record_expr p | LessThan -> parse_jsx p - | Percent -> - let extension = parse_extension p in + | Percent -> ( + let (({Location.txt = name; loc = name_loc}, payload) as extension) = + parse_extension p + in let loc = mk_loc start_pos (Parser.position p) in - Ast_helper.Exp.extension ~loc extension + match (Ast_raw.kind_of_name name, Ast_raw.of_payload payload) with + | Some kind, Some source -> Ast_helper.Exp.raw ~loc ~name_loc kind source + | _ -> Ast_helper.Exp.extension ~loc extension) | Underscore as token -> (* This case is for error recovery. Not sure if it's the correct place *) Parser.err p (Diagnostics.lident token); @@ -6458,11 +6462,16 @@ and parse_structure_item_region pending_structure_items p = parse_newline_or_semicolon_structure p; let loc = mk_loc start_pos (Parser.position p) in Some (Ast_helper.Str.attribute ~loc attr) - | PercentPercent -> - let extension = parse_extension ~module_language:true p in + | PercentPercent -> ( + let (({Location.txt = name; loc = name_loc}, payload) as extension) = + parse_extension ~module_language:true p + in parse_newline_or_semicolon_structure p; let loc = mk_loc start_pos (Parser.position p) in - Some (Ast_helper.Str.extension ~attrs ~loc extension) + match (name, Ast_raw.of_payload payload) with + | "raw", Some source -> + Some (Ast_helper.Str.raw ~attrs ~loc ~name_loc source) + | _ -> Some (Ast_helper.Str.extension ~attrs ~loc extension)) | token when Grammar.is_expr_start token -> let position = Parser.position p in let exp = parse_expr p in @@ -7438,37 +7447,6 @@ and parse_extension ?(module_language = false) p = (Diagnostics.message "The %re extension has been removed. Use a regexp literal such as \ /abc/i."); - let payload = - match (attr_id.txt, payload) with - | ( ("raw" | "ffi"), - Parsetree.PStr - [ - ({ - pstr_desc = - Pstr_eval - ( ({pexp_desc = Pexp_constant (Pconst_string payload)} as - expression), - eval_attrs ); - } as item); - ] ) -> - Parsetree.PStr - [ - { - item with - pstr_desc = - Pstr_eval - ( { - expression with - pexp_desc = - Pexp_constant - (Pconst_raw_source - (String_literal.string_source payload)); - }, - eval_attrs ); - }; - ] - | _ -> payload - in (attr_id, payload) (* module signature on the file level *) diff --git a/compiler/syntax/src/res_parens.ml b/compiler/syntax/src/res_parens.ml index 325a108f35..e6a8362f31 100644 --- a/compiler/syntax/src/res_parens.ml +++ b/compiler/syntax/src/res_parens.ml @@ -106,7 +106,7 @@ let unary_expr_operand expr = pexp_desc = ( Pexp_assert _ | Pexp_fun _ | Pexp_constraint _ | Pexp_coerce _ | Pexp_setfield _ | Pexp_extension _ (* readability? maybe remove *) - | Pexp_object_literal _ (* ({"a": 1})["a"] *) + | Pexp_raw _ | Pexp_object_literal _ (* ({"a": 1})["a"] *) | Pexp_object_set _ (* (o["x"] = v)["y"] *) | Pexp_match _ | Pexp_try _ | Pexp_while _ | Pexp_for _ | Pexp_for_of _ | Pexp_for_await_of _ | Pexp_ifthenelse _ | Pexp_ternary _ ); @@ -271,9 +271,10 @@ let field_expr expr = | { pexp_desc = ( Pexp_assert _ | Pexp_extension _ (* %extension.x vs (%extension).x *) - | Pexp_regexp _ | Pexp_object_literal _ (* ({"a": 1})["a"] *) - | Pexp_fun _ | Pexp_constraint _ | Pexp_coerce _ | Pexp_setfield _ - | Pexp_match _ | Pexp_try _ | Pexp_while _ | Pexp_for _ | Pexp_for_of _ + | Pexp_raw _ | Pexp_regexp _ + | Pexp_object_literal _ (* ({"a": 1})["a"] *) | Pexp_fun _ + | Pexp_constraint _ | Pexp_coerce _ | Pexp_setfield _ | Pexp_match _ + | Pexp_try _ | Pexp_while _ | Pexp_for _ | Pexp_for_of _ | Pexp_for_await_of _ | Pexp_ifthenelse _ | Pexp_ternary _ ); } -> Parenthesized @@ -334,9 +335,9 @@ let jsx_prop_expr expr = Parsetree.pexp_desc = ( Pexp_ident _ | Pexp_constant _ | Pexp_field _ | Pexp_construct _ | Pexp_variant _ | Pexp_array _ | Pexp_pack _ | Pexp_record _ - | Pexp_object_literal _ | Pexp_extension _ | Pexp_letmodule _ - | Pexp_letexception _ | Pexp_open _ | Pexp_sequence _ | Pexp_let _ - | Pexp_tuple _ ); + | Pexp_object_literal _ | Pexp_extension _ | Pexp_raw _ + | Pexp_letmodule _ | Pexp_letexception _ | Pexp_open _ + | Pexp_sequence _ | Pexp_let _ | Pexp_tuple _ ); pexp_attributes = []; } -> Nothing diff --git a/compiler/syntax/src/res_parsetree_viewer.ml b/compiler/syntax/src/res_parsetree_viewer.ml index 9cfa195424..b11c5a0859 100644 --- a/compiler/syntax/src/res_parsetree_viewer.ml +++ b/compiler/syntax/src/res_parsetree_viewer.ml @@ -236,8 +236,6 @@ let is_huggable_expression expr = | Pexp_constant (Pconst_string payload) when is_multiline_text (String_literal.string_source payload) -> true - | Pexp_constant (Pconst_raw_source source) when is_multiline_text source -> - true | _ -> false let is_huggable_rhs expr = diff --git a/compiler/syntax/src/res_printer.ml b/compiler/syntax/src/res_printer.ml index 910d699fec..3cb9cd6d65 100644 --- a/compiler/syntax/src/res_printer.ml +++ b/compiler/syntax/src/res_printer.ml @@ -641,17 +641,6 @@ let interleave_template_parts sources values = in loop Doc.nil sources values -let raw_source_fits_double_quotes source = - let rec loop index = - if index >= String.length source then true - else - match String.unsafe_get source index with - | '"' -> false - | '\\' -> index + 1 < String.length source && loop (index + 2) - | _ -> loop (index + 1) - in - loop 0 - let print_constant c = match c with | Parsetree.Pconst_integer (s, suffix) -> ( @@ -665,12 +654,6 @@ let print_constant c = print_string_contents (String_literal.string_source payload); Doc.text "\""; ] - | Pconst_raw_source source -> - let delimiter = - if raw_source_fits_double_quotes source then "\"" else "`" - in - Doc.concat - [Doc.text delimiter; print_string_contents source; Doc.text delimiter] | Pconst_char {source} -> Doc.concat [Doc.text "'"; Doc.text source; Doc.text "'"] | Pconst_float (s, _) -> Doc.text s @@ -756,6 +739,15 @@ and print_structure_item ~state ?(protect_regexp = false) | Pstr_attribute attr -> clear_pending (); fst (print_attribute ~state ~standalone:true attr cmt_tbl) + | Pstr_raw {name_loc; source; attrs} -> + clear_pending (); + Doc.concat + [ + print_attributes ~state attrs cmt_tbl; + print_extension ~state ~at_module_lvl:true + (Ast_raw.to_extension ~name:"raw" ~name_loc source) + cmt_tbl; + ] | Pstr_extension (extension, attrs) -> clear_pending (); Doc.concat @@ -3548,6 +3540,12 @@ and print_expression ~state (e : Parsetree.expression) cmt_tbl = ]) | Pexp_extension extension -> print_extension ~state ~at_module_lvl:false extension cmt_tbl + | Pexp_raw {kind; name_loc; source} -> + print_extension ~state ~at_module_lvl:false + (Ast_raw.to_extension + ~name:(Ast_raw.extension_name kind) + ~name_loc source) + cmt_tbl | Pexp_regexp {pattern; flags} -> Doc.text ("/" ^ pattern ^ "/" ^ flags) | Pexp_template {source_segments; values} -> print_template_literal ~state ~source_segments ~values cmt_tbl diff --git a/tests/ounit_tests/ounit_ast_mapper0_tests.ml b/tests/ounit_tests/ounit_ast_mapper0_tests.ml index f0ebbb5b86..c20a3e50ee 100644 --- a/tests/ounit_tests/ounit_ast_mapper0_tests.ml +++ b/tests/ounit_tests/ounit_ast_mapper0_tests.ml @@ -1668,21 +1668,159 @@ let test_string_literals_roundtrip_through_ast0 _ = assert_string_expr ~expected_source:{|\x61|} ~expected_semantic:"a" (map_expr0 (map_expr_to0 source_string)) +let test_raw_nodes_roundtrip_through_ast0 _ = + (* [%raw("x")] and [%ffi(`y`)] reach v0 as the extensions they are written + as, and come back unchanged. *) + let string_source : Parsetree.raw_source = + Raw_string {txt = "x"; loc = source_loc 5 8} + in + let template_source : Parsetree.raw_source = + Raw_template {txt = "a\\`b"; loc = source_loc 5 10} + in + let raw kind source = + Ast_helper.Exp.raw ~loc:(source_loc 0 12) ~name_loc:(source_loc 0 4) kind + source + in + List.iter + (fun (expression : Parsetree.expression) -> + let expression0 = map_expr_to0 expression in + (match expression0.pexp_desc with + | Pexp_extension + ( {txt = "raw" | "ffi"; loc = name_loc}, + PStr + [ + { + pstr_desc = + Pstr_eval + ( { + pexp_desc = + Pexp_constant (Pconst_string (_, Some "js")); + }, + [] ); + }; + ] ) -> + OUnit.assert_equal (source_loc 0 4) name_loc + | _ -> assert_failure "Expected a raw extension on the ast0 wire"); + OUnit.assert_equal expression.pexp_desc (map_expr0 expression0).pexp_desc) + [ + raw Raw_expression string_source; + raw Raw_function string_source; + raw Raw_expression template_source; + ]; + (* A template keeps its spelling, so its JavaScript is decoded once. *) + match + (map_expr0 (map_expr_to0 (raw Raw_expression template_source))).pexp_desc + with + | Pexp_raw {source} -> + OUnit.assert_equal ~printer:(Printf.sprintf "%S") "a`b" + (Ast_raw.js_source source) + | _ -> assert_failure "Expected a raw node" + +let test_raw_structure_item_roundtrips_through_ast0 _ = + let item = + Ast_helper.Str.raw ~loc:(source_loc 0 20) ~name_loc:(source_loc 7 12) + ~attrs:[attr "attr" (Parsetree.PStr [])] + (Raw_string {txt = "var x"; loc = source_loc 13 20}) + in + let item0 = + Ast_mapper_to0.default_mapper.structure_item Ast_mapper_to0.default_mapper + item + in + (match item0.pstr_desc with + | Pstr_extension (({txt = "raw"; loc}, PStr [_]), [({txt = "attr"}, _)]) -> + OUnit.assert_equal (source_loc 7 12) loc + | _ -> assert_failure "Expected a %%raw extension on the ast0 wire"); + OUnit.assert_equal item.pstr_desc + (Ast_mapper_from0.default_mapper.structure_item + Ast_mapper_from0.default_mapper item0) + .pstr_desc + +let test_raw_source_prints_with_fitting_quotes _ = + (* A source with a double quote, e.g. from a PPX, prints as a template. *) + let payload_desc txt = + match + Ast_raw.to_extension ~name:"raw" ~name_loc:loc + (Raw_string {txt; loc = Location.none}) + with + | _, PStr [{pstr_desc = Pstr_eval ({pexp_desc}, [])}] -> pexp_desc + | _ -> assert_failure "Expected a single payload expression" + in + (match payload_desc "1 + 2" with + | Pexp_constant (Pconst_string _) -> () + | _ -> assert_failure "Expected a double-quoted string"); + match payload_desc {|"img"|} with + | Pexp_template {source_segments = [{txt = {|"img"|}}]} -> () + | _ -> assert_failure "Expected a template" + +let test_raw_source_template_fallback_reparses _ = + (* A source that needs a template escapes the template's own syntax, so it + parses back to the same JavaScript. *) + List.iter + (fun txt -> + let structure = + [ + Ast_helper.Str.value ~loc Nonrecursive + [ + Ast_helper.Vb.mk ~loc + (Ast_helper.Pat.var ~loc (Location.mknoloc "v")) + (Ast_helper.Exp.raw ~loc ~name_loc:loc Raw_expression + (Raw_string {txt; loc})); + ]; + ] + in + let printed = + Res_printer.print_implementation structure ~comments:[] ~width:80 + in + let parsed = + Res_driver.parse_implementation_from_source + ~display_filename:"RawTemplateFallback.res" ~source:printed + in + match parsed.parsetree with + | [ + { + pstr_desc = + Pstr_value (_, [{pvb_expr = {pexp_desc = Pexp_raw {source}}}]); + }; + ] + when not parsed.invalid -> + OUnit.assert_equal ~printer:(Printf.sprintf "%S") txt + (Ast_raw.js_source source) + | _ -> assert_failure ("Printed raw source does not reparse: " ^ printed)) + [ + {|"${x}"|}; + {|"a`b"|}; + {|"\"q\" `r` ${s}"|}; + {|"a\${b}"|}; + {|"\\`"|}; + {|"a\n" + "\|}; + "\"line\\\ncontinued\""; + ] + +let test_attributed_raw_template_reprints_after_ast0_roundtrip _ = + let source = {|let a = %raw(@foo `"x"`) +let b = %ffi(@foo `(a) => "\`"`)|} in + let parsed = + Res_driver.parse_implementation_from_source + ~display_filename:"AttributedRawTemplate.res" ~source + in + OUnit.assert_bool "expected valid ReScript source" (not parsed.invalid); + let round_tripped = + Ast_mapper_from0.default_mapper.structure Ast_mapper_from0.default_mapper + (Ast_mapper_to0.default_mapper.structure Ast_mapper_to0.default_mapper + parsed.parsetree) + in + let print structure = + Res_printer.print_implementation structure ~comments:[] ~width:80 + in + OUnit.assert_equal ~printer:(Printf.sprintf "%S") (print parsed.parsetree) + (print round_tripped) + let assert_raw_extension_payload ~name ~expected expression = match expression.Parsetree.pexp_desc with - | Pexp_extension - ( {txt}, - PStr - [ - { - pstr_desc = - Pstr_eval - ({pexp_desc = Pexp_constant (Pconst_raw_source actual)}, _); - }; - ] ) -> - OUnit.assert_equal name txt; + | Pexp_raw {kind; source = Raw_string {txt = actual}} -> + OUnit.assert_equal name (Ast_raw.extension_name kind); OUnit.assert_equal ~printer:(Printf.sprintf "%S") expected actual - | _ -> assert_failure "Expected a raw extension string payload" + | _ -> assert_failure "Expected a raw node with a string payload" let test_raw_extension_payloads_roundtrip_through_ast0 _ = let encoded = {|'\\n'|} in @@ -2356,6 +2494,16 @@ let suites = "constructor_runtime_tag_reaches_ast0_as_an_attribute" >:: test_constructor_runtime_tag_reaches_ast0_as_an_attribute; "ppx_context_on_the_wire" >:: test_ppx_context_on_the_wire; + "raw_nodes_roundtrip_through_ast0" + >:: test_raw_nodes_roundtrip_through_ast0; + "raw_structure_item_roundtrips_through_ast0" + >:: test_raw_structure_item_roundtrips_through_ast0; + "raw_source_prints_with_fitting_quotes" + >:: test_raw_source_prints_with_fitting_quotes; + "raw_source_template_fallback_reparses" + >:: test_raw_source_template_fallback_reparses; + "attributed_raw_template_reprints_after_ast0_roundtrip" + >:: test_attributed_raw_template_reprints_after_ast0_roundtrip; "fixed_argument_roundtrips_through_ast0" >:: test_fixed_argument_roundtrips_through_ast0; "fixed_argument_is_external_only" diff --git a/tests/syntax_tests/data/ast-mapping/RawExtensions.res b/tests/syntax_tests/data/ast-mapping/RawExtensions.res new file mode 100644 index 0000000000..3b21442b87 --- /dev/null +++ b/tests/syntax_tests/data/ast-mapping/RawExtensions.res @@ -0,0 +1,20 @@ +let add = %raw("(a, b) => a + b") +let template = %raw(`function (a) { + return "x" + a +}`) +let escaped = %raw(`"a\`b\${c}"`) +let fn = %ffi("(a) => a + 1") +let applied = %ffi(`(a, b) => a * b`)(2, 3) +let attributed = @foo %raw("1") +let leftAsExtension = %raw({"x"}) +let bracedEscapes = %raw({"\"a\tb\""}) + +%%raw("var top = 1") + +@attr +%%raw(` + var multi = 2 +`) + +/** Doc comment */ +%%raw("var documented = 3") diff --git a/tests/syntax_tests/data/ast-mapping/expected/RawExtensions.res.txt b/tests/syntax_tests/data/ast-mapping/expected/RawExtensions.res.txt new file mode 100644 index 0000000000..3b21442b87 --- /dev/null +++ b/tests/syntax_tests/data/ast-mapping/expected/RawExtensions.res.txt @@ -0,0 +1,20 @@ +let add = %raw("(a, b) => a + b") +let template = %raw(`function (a) { + return "x" + a +}`) +let escaped = %raw(`"a\`b\${c}"`) +let fn = %ffi("(a) => a + 1") +let applied = %ffi(`(a, b) => a * b`)(2, 3) +let attributed = @foo %raw("1") +let leftAsExtension = %raw({"x"}) +let bracedEscapes = %raw({"\"a\tb\""}) + +%%raw("var top = 1") + +@attr +%%raw(` + var multi = 2 +`) + +/** Doc comment */ +%%raw("var documented = 3") diff --git a/tests/tests/src/raw_output_test.mjs b/tests/tests/src/raw_output_test.mjs index c61d251d39..ddb1b46f7a 100644 --- a/tests/tests/src/raw_output_test.mjs +++ b/tests/tests/src/raw_output_test.mjs @@ -12,7 +12,13 @@ console.log(1); var issue6236 = `${"hello"}`; ; +let bracedRaw = "braced"; + +var bracedItem = "item"; +; + export { mk, + bracedRaw, } /* Not a pure module */ diff --git a/tests/tests/src/raw_output_test.res b/tests/tests/src/raw_output_test.res index fd4b7f4599..91b5372b2c 100644 --- a/tests/tests/src/raw_output_test.res +++ b/tests/tests/src/raw_output_test.res @@ -18,3 +18,7 @@ Console.log((() => 1)()) %%raw(` var issue6236 = \`\${"hello"}\`; `) + +// A braced string payload is an ordinary string, so its escapes are decoded. +let bracedRaw = %raw({"\"braced\""}) +%%raw({"var bracedItem = \"item\";"})