From e39be71a54a4264032115ab3b31bdd86845a966b Mon Sep 17 00:00:00 2001 From: Laurenz Date: Tue, 12 Sep 2023 11:45:24 +0200 Subject: [PATCH] Fix newline parsing behaviour in markup --- crates/typst-syntax/src/parser.rs | 57 ++++++++++++++++++------------ tests/ref/bugs/newline-mode.png | Bin 0 -> 7771 bytes tests/typ/bugs/newline-mode.typ | 24 +++++++++++++ 3 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 tests/ref/bugs/newline-mode.png create mode 100644 tests/typ/bugs/newline-mode.typ diff --git a/crates/typst-syntax/src/parser.rs b/crates/typst-syntax/src/parser.rs index 313d5ea3..bd184b90 100644 --- a/crates/typst-syntax/src/parser.rs +++ b/crates/typst-syntax/src/parser.rs @@ -571,13 +571,13 @@ fn code(p: &mut Parser, stop: impl FnMut(&Parser) -> bool) { fn code_exprs(p: &mut Parser, mut stop: impl FnMut(&Parser) -> bool) { while !p.eof() && !stop(p) { - p.stop_at_newline(true); + p.enter_newline_mode(NewlineMode::Contextual); let prev = p.prev_end(); code_expr(p); if p.progress(prev) && !p.eof() && !stop(p) && !p.eat_if(SyntaxKind::Semicolon) { p.expected("semicolon or line break"); } - p.unstop(); + p.exit_newline_mode(); if !p.progress(prev) && !p.eof() { p.unexpected(); } @@ -593,7 +593,7 @@ fn code_expr_or_pattern(p: &mut Parser) { } fn embedded_code_expr(p: &mut Parser) { - p.stop_at_newline(true); + p.enter_newline_mode(NewlineMode::Stop); p.enter(LexMode::Code); p.assert(SyntaxKind::Hashtag); p.unskip(); @@ -611,12 +611,8 @@ fn embedded_code_expr(p: &mut Parser) { code_expr_prec(p, true, 0, false); // Consume error for things like `#12p` or `#"abc\"`.# - if !p.progress(prev) { - if p.current().is_trivia() { - // p.unskip(); - } else if !p.eof() { - p.unexpected(); - } + if !p.progress(prev) && !p.current().is_trivia() && !p.eof() { + p.unexpected(); } let semi = @@ -627,7 +623,7 @@ fn embedded_code_expr(p: &mut Parser) { } p.exit(); - p.unstop(); + p.exit_newline_mode(); } fn code_expr_prec( @@ -772,7 +768,7 @@ pub(super) fn reparse_block(text: &str, range: Range) -> Option SyntaxKind { - p.stop_at_newline(false); + p.enter_newline_mode(NewlineMode::Continue); let m = p.marker(); p.assert(SyntaxKind::LeftParen); @@ -901,7 +897,7 @@ fn collection(p: &mut Parser, keyed: bool) -> SyntaxKind { } p.expect_closing_delimiter(m, SyntaxKind::RightParen); - p.unstop(); + p.exit_newline_mode(); if parenthesized && count == 1 { SyntaxKind::Parenthesized @@ -1442,10 +1438,20 @@ struct Parser<'s> { current: SyntaxKind, modes: Vec, nodes: Vec, - stop_at_newline: Vec, + newline_modes: Vec, balanced: bool, } +/// How to proceed with parsing when seeing a newline. +enum NewlineMode { + /// Stop always. + Stop, + /// Proceed if there is no continuation with `else` or `.` + Contextual, + /// Just proceed like with normal whitespace. + Continue, +} + #[derive(Debug, Copy, Clone, Eq, PartialEq)] struct Marker(usize); @@ -1462,7 +1468,7 @@ impl<'s> Parser<'s> { current, modes: vec![], nodes: vec![], - stop_at_newline: vec![], + newline_modes: vec![], balanced: true, } } @@ -1597,13 +1603,13 @@ impl<'s> Parser<'s> { } } - fn stop_at_newline(&mut self, stop: bool) { - self.stop_at_newline.push(stop); + fn enter_newline_mode(&mut self, stop: NewlineMode) { + self.newline_modes.push(stop); } - fn unstop(&mut self) { + fn exit_newline_mode(&mut self) { self.unskip(); - self.stop_at_newline.pop(); + self.newline_modes.pop(); self.lexer.jump(self.prev_end); self.lex(); self.skip(); @@ -1654,8 +1660,15 @@ impl<'s> Parser<'s> { self.current = self.lexer.next(); if self.lexer.mode() == LexMode::Code && self.lexer.newline() - && self.stop_at_newline.last().copied().unwrap_or(false) - && !matches!(self.lexer.clone().next(), SyntaxKind::Else | SyntaxKind::Dot) + && match self.newline_modes.last() { + Some(NewlineMode::Continue) => false, + Some(NewlineMode::Contextual) => !matches!( + self.lexer.clone().next(), + SyntaxKind::Else | SyntaxKind::Dot + ), + Some(NewlineMode::Stop) => true, + None => false, + } { self.current = SyntaxKind::Eof; } diff --git a/tests/ref/bugs/newline-mode.png b/tests/ref/bugs/newline-mode.png new file mode 100644 index 0000000000000000000000000000000000000000..d4b6c6d85ed927cd490bbb8a3bdaf8faccf58fea GIT binary patch literal 7771 zcmb7}byU>vm&Z|*90|!mx;qAy21)5g1Oxk64DLQ-Q6NccMR!1 z?jO5*c7J>J>>SRV&pe;L@xHHn2~k&l`3Q#s2MrDFk%GL8CK}qk2VlD$>puA2IYh0a zp;6N-$Vh6t&+VmoyiqY&Zk@Iy!;VRY;cq=nA}U#ZoJB0)A|0$8^&yOVzbP*4TIPqs zXVxs4+{Y@KtXhTR3v4QnLpC&AD5kGH4m_7`cQ2j_)?s!xwH*AuaU5Fm`c^R{e%&x# zc5%mr1h@PbpB3DX_g3s5^q`^9qN9C;prOIg(Egu4K1N&HzW-Lh!%$kEho7a+STFjl zkv)2qVRxbKB#6dm#lW3qO-@FYpbO=Cb#`6Xc6wZy^?=lX{nF6JvW|(gtG09AB%BDx ztfXTe_iMD8kt`~w-Xxr+1*aw1Q}z&f883u4`YM8}QI}*EY^ZQWNU^DTI>w$jJqhQl zVLg)Cwf4OH8frso5ZV|i!E2vUmZG~wV0+elZmT05AYG}WVVkMmcT$qqN;g*&sH^k( zqzi0Rm!}xzTFqy~bsy%*1zD$MWc0DBc9Ca9l@GBhT>kEhPDx?s=Du5LLvLQ6zn#9J zsKzz3`IYazxn$R7UOOOE(<6&#h6pihn2|Mly)Ftp`?_={OTI=FgL!7pNh-RU-N;L7 z0PZ4uY|a|Sr8BL~5qFHrGfXp)fSB!m-hMuA^1z9twC>aNLfH>n&mdUiw;>jhLlqtB zFs}IHbcVQNM$)db8O?xa_CJ^9DrPhf-Y*}=ccZF(EMdwOG(QtJKJlfS3T%>I%t`2C zNK3b~Y2{TK z)2(9>W}8){!CG`pb{axe-#xvi3kl#m)gPL-oTDysQK@o#s8%Vaz4K)UshRFITjSe! zB-SO!De_NyN)bOa^c`-Mm^Q!C_|kOq1PuF%jV7F zGkNRz^-|rch;JKvyXWvjn@J6O(6SZ&^jF)?=j7%##3OqS56!VIX9DNrh9A|YtQ=Ok z{)~X@%Ff^EJR&)L-178-&DCFjUSAR;I`fImT%~nKFkuF*^jO310n&V>h1t?7qHozg zT5m73D)&mQ=8ZosM)|}e-0@)_A&adG3#nNJ_%!0qx)>W)W7F|%d>b9gL4*O5rMT~x zc#YNh!9nedi&S)Rsi`+i=B%u&D-ME8OiWUj*e0|0IIAbVhVH5jWfm8Qx34z(-0Ii< z*%#)i^3R-?pffQvqzY>99T*_C^T1o?pE^qYgN;2uHz!D`s;N1){3dX^)L4sneQ)oU z$z0H31x>1;{b)h$pWJV<{odyo2j-`_3uH`%)3K=!hgbXJIk4!6h=^A28d$P@(!6|> z^);ky&0M6NnpUvQ%*+U-!pL|??K}|UCG4|`5`DN|S9M(8BkHQFN$H;2%#^j%LCDRX zJ$t62GRVW$zQ5Q&DCOhh^WzE|r1q5nDaYKou9FUkCY`qzYfrrPC@?WGkt0WI-2vAJ zuOp8RA?}xuEkv4Rv?xg&dhE)cSJ= zTc~q-sjSQ?VlZNXQP&$Zz0fL3n(Qw`Bde#E+-X`>jZ>krJU5l1l|Yz4b;!z7e0!Ex zGCDndu-xM7;c@vJ71h73Xru#o`zqkOJHyAvS8i8j_%J_E_pFHi+Sa9cMZs>VF@)gw z`1tC&|0ADKt;1@%m{+s1L}6`hQWh%>4bA9Q{Ua*j(~D(avb1AP{YrT?H8tnH3a<-0 zbMy7;>S|vkBp@Jw$_RN!NlZ#wUso40Bw=W5e6YQ(Z)nIREc?ODjm*>ju9SeB9F<(e zrV)1!ty%bA%JYp@!TTbvVvq5$sj)>Q8TXr>{(gTdRw)?L;&S*yTpM{1{A$RRg3bNM z@zt1KBn+gQ;!dcgP@lt+I8onz52wnPb=H$2dXa2K^Mq_=A9L=eZSTGL@hGn6SDd;o z6ji6On%9-@N^q9g68XBq41YC{$mijv=u?18i1s{MeI_vQAua>~RV+PLX(;k=$-Cb` z6UOzd({5z++Jc7SeUf#}b*T{a64%3TL3mrBIrR|)@~Y$w_Wj^!h$IiL7#GP2bJ3|ZA=ofDe9hXxkj)s-nc%6H9&eP|;ih{&esit#MsK4n@T=Iy zfD`q+_+ILXndoQkx8XWza*Oh=gb5JNYHjx+DkCTKFfQ3FdBW<;(rPS$kRI%xq)hdl z^D06D(V`hD6{D7y<=0+(T~8Vlt9*;-xVa;<3JMBX9ag@?WEN$LHVK9tq+aO2t=j+8 z@ogkFh{#A{G#A_tI)HM_D)N)La-ny*FEx79*VnJK3eZYFEB!E>=JLx?@u90kn1k7d zZIQ7N1_fcvC;aoz%a`fxcLfm3;q>_E=*=nqK-zZ8xK$OMbct#I#?`O{Csg`T>PdD+ zi^s&o7$Kl_y-3`3}T*`^Wc;@@nqxf>yL+ z*$Pz=lUhTMD!p7&w*rgCSv6+N(|Qj{j#TT4;T(hE(X?1RMvX79X6NVqG3dF_txHDQ z-uC<%G*yb9jQ2?k4Xu|2lFfmJOM3BBqtb>}U)zEzkXllRl#4XS~4p z%_DmCPujE4&pMRm5?l}DOibvcdb+wWo)TC?<<-wdIDI`sy!^87HyB@=e&SeJ$N}qR;J4gadv$M zLVsoheWyce>*xgNq~k|CBVA<%e;f;WDit(ROZg;BSV@h!xew_kN{uB<;w{q>Q)0iJ zwvFuNF#bBLKh9kdvRSOxQBc69L}9>b+bRXFIE{DdTRL?s8e)>s@82hodKpPc_{yZn zBfjCj^QDpBGDd8Cd;r1YpRy$kgc4Oe_|SP0F8v^{F5pwDr#%pJw5A#!{adB~3XeugT;Tj09L_K|9Q^eAE*xi0sqzKqr*pB z6xu@MJs4oc9sr9#l0X>)xcJ`|{!fNnZiEyT7IJfQ&)3-Fl7hzJXB@9bE@ z;cvsin=dXbC@U+M7&WoeH$0=IZSCr^INP4&;^JDqqSG`mXm~$evazv2CE|MX$S0Du zFFGly)^$%GIl@BOW@2K(&d!dNi9)%DJ9>Iv<*9rjkYZ6yC5Tzv-r8d0;NSpY_po`V zX4%Kd)%8v}S>XBeqQb&}pdhnwF;UTKyG0xrBEdov4ri3=>F)kbG&VOoONSgAAMfoZ z<1wN|4yB2(2z$AnYz%r|pWiMlkByDN5U4}hFjDOFUw}F;_7{m9*0Ntl?ax((-4zrT zT3cFT!mx}3S=7>LkqB>Z3A()iTrvVFkY=P)2Fn6J=CY?RsOG{A}vL z^UPLQ@+VeSRrU7uQ42bt!BLzr)b({uRaF2I`TF&1N}&%qpib4PXlZ?}skvBf4=Ocm zIKR8Sw(E^fO+{=BK((~AFfoJr`V`x6U+L=p-q^6Puy6&z*U=%3k4S63;?$+tHr!yumE5-0AW*4A76mP3c}9TOAI z14(>9Z(~+}1mNyWmkNMlURD+z9sRn|ov%q!Qu0$#QBg&OlZ8cIa`J3X48s=tbvA5f zW`^^1*$Mypg)*+lmtXXxHvfUUu|q{e-C~k zBOxK7p()MI=6~Dz5jdLn%U?uTT3X^UZg&3gAvQYt?cXRBH1CW1`0-<09I>I{B9z}k z`_-$2xHzCC@mZ{3Ma=7*Qp9zjt<7Yn%!HG4XR*QUl(5CTKjCa=nuUo;%>D2Mcna{~ zlpX8q>mYsdSwrp4Q-xjLZH{DuI8;j)1yUgBdAd24t3)VOW4HK&e0qA?%hR*3r-zk; z18jJDcuY=B#Zs&^95na$^?3qo{v+uxZ%9Qh8Ts1p;S4R}&h-yl>0g1yhleWzQ0=Py{$2CctEs`kh#^Ha zwWx&5%uJK99EHfp$X{y>ZU?NPSxO*n=fCQf0>d5iJ%65%RS@w+U^0+-oTlbGk}Ki%1p)M6A+ zXb7K0#Vi(Gz4p_#wl*v}U0vP3FU=eD3Rp6PfPT_hSs5bL)7rW+Q*M@!kbsF#y*r+d z?~SvoA2|6{7qENkrQxerudp&ffp;;?0S`-$++S)^;sH9tVDEEt)O`;fgI7S@X#*m) z39^lRt-^9xlep7njqe&_XJ>Rx)`zSXnPzHv$DFDmf41=1fQLq_3PmUj!fpagthc9FVoODgPbi1gPWQ86a3s^8S&>;R8-s>jsY+Wr`1tb zF8=oIn>6%%Z~mo*wzf9$twFlDw_?pSi*nLGg@wHX9>DHcnQjM5lM@r_#8qb9cpG#7 z-kqgO_$~!NA|fJAV?ofA8a379LjC;wt}YJLG$xMM`-Ld+^sL};Bmv0NpY!Rr)(u&7FCPrhq*&Fml>>M0yq52K3TuxkH2J=`$#bf@s zK;J z!Q;nL8WWviWLr!ip0P3~CbY-OzX&>J)z%6^T^Z36PYXfjric^tzJ4UNxjCAB`cDzq zb+U9%OiYZ5Dh)3zCONsz=hkzK-|pSJ*w3Fo$HrzE-~#6|HKl}yhXZAxASWNe7O-28 zr|AG?-qh5TL$`$SkvDgui2LDNBmp!8Ojhg$# zvv_G|ZJjLYaoqCf*531)kU|Cok5k2;9dxS>4I=e$gcjs0O-?s>8Qt*}rCT)|~$YBbw` zj*d<`5HdP7)$D$hm6^F(r1LeYI~C*(xc5LZ|Jms&NTC|vKX<^0-1?Ql3)ciBBuqg0 z*x2e!J44s|6Q4k#&z?RVfvFRl+t?u1x+0{5@OpcDj|_uFy|0`z#J%s%N`33<>$&y5 z!w;64au9!E?*gQYn~RHSxxs3bzL_1@Ow?stdt-*U zhxP#!=u~`4N=kZqdP<67eJo%AkgQt(x7gTD-@aE;QUY1@_U&6AAMvz={`FWcpwK5L zCm<`Nzbjr7ZB$i&x`2s+F=kdwH8wFp42AaEzy+Q?V`65;c=(Wy=Y$ZjAs98FqoY@R z|2}7V1`M^bveMWn8tw=n$j*-2i7Tpm%|Q^D7(qq~gr^J`EHyQ?wY7C~)57Mr{h zsTu5SYzF%Ky^eo{WMt47(v0a|0edB(Yvp+6jQc?jMg60#mn{jX0Ca5$6PejqSWKA& z;Q;1FwkPqKC6SHFgn~*326WN&+`_`1+Y^N};t0Uyfx}XOe1Ml&R8nLvlw~47ID32F z0^oY|DB^ri4iGy5o?3=@nuPCNIFvt6Eu-0C^#>4+FbkpHT!9Y^M1RTK*Z02z{C{)+ z{{!v+>z)+m9;hYm{yu80``J(*Xp>6@pbn0tsdDmL+FNZ@e z4wnDyHyjX1<*Q|onpTsNk^;@&=Vd3109+6hgc&j~OMm(B!GqTZ2Dq^j$<}RvvikZ$ zvkE}_RJgOJTk=LBPkk07>~+rp70k2)3#snv`W!n&&aX60(9tiwI7b@B&+m1*>9{q< zB)HJhYoUoih)VVJ_9_H>sAh;a?IcMI4GlRfkNwTA{(ewr-qd|C2Ss=N=z zVCCjEYH+nxS66p&sVXdF`!dKpC`3XMUt)>=MZ#B{KalZ|t2a71Spl<_loVhmvxbZa zpmlf95;6{OY%vChg&oY4uK^bHzB)CL+g^7Nd_Pr8NJKO~Huke0{^#x%uD^pz#*J{^ z)&MCNnssw~3pyIsP?t+b-{dh}qDK0T_T(T@IG}PyMuV!Rq zq}lr#CT;N9BB`N4qziR(d2;{x;imdGXTib1Knw*11yAe`8~{^U;h5jyny`@ zB&cDtZ)IcSx!r=Gr+!4PQsS7C+EMRYTU#-Ql@~C<*G5L-mP3@^Z@^&HtUGd#7gHRl z+x6*I1*oD!fo~c_{MQp>OhA7L9+a3dZV@<(PzugkEC#&&N!(w)pBLT}O~UaCb$RLG z;Q?SoMOIc;O3H7*4~*zR%Lr0;6T5U%Aq{AKQAx>AB07JqXhc=jS%F5ju!v*ImoE%R z6n&?!3sh80tUKb#%MHFO*7EXlEV|}<+!hs1KRZGOLY~|sjHQ#1prNY-#oo>B0u=E# z&4}5_$;rjVv;n$S7HL`8fY{i_QdS!S$)}w5mzS3SR}#=agZ@-GT{O8gzpZ0rgn&*) zjfVpVw$SWtprH|?Y|nxPg+c)($)(CgQ7`IE-b1KBfByVQAO+Bx*x_kN_ZpZxMM3#7 z=wz6KfUIP*7FAH9fnn7Q_Izg+pacO=5I3 zL0RlQXmce5jLESwt;h4Q>CwGq!Bqi?#6S~4RoG2(tre?lNfl;iN2YP;MRRo$v1|&T_z3>|mr;ULqqX!R~Eixnx4b$xHHo!e#S!xzb%onR~ z!@5fF1(23*wC)O34ugFqm9ZpyH+vBky3fl!OPY!iDMJWG?7 zm&Y>3OH1o-s1@jKZ_hSv@iAN&eL+M?xx2Qu_WSqmot>Ta_0Cky@^(pT&`4k1KY8i# z=T7W$@9s}fR206`@fIE%D=TQO8MJa-gHuxG041$BkXb{U838