Stabel

Check-in [f0c47e6da7]
Login
Overview
Comment:Implemented multiline strings except for indentation handling.
Timelines: family | ancestors | descendants | both | strings
Files: files | file ages | folders
SHA3-256: f0c47e6da789cd7c4c41ec64150bf62578f739c067fad453c66d6580793d2253
User & Date: robin.hansen on 2021-09-05 09:36:21
Other Links: branch diff | manifest | tags
Context
2021-09-05
09:54
Strip indentation from multiline strings. check-in: b65fbd80ac user: robin.hansen tags: strings
09:36
Implemented multiline strings except for indentation handling. check-in: f0c47e6da7 user: robin.hansen tags: strings
2021-09-04
11:41
Error handling for string parser. check-in: 3dd9b18af1 user: robin.hansen tags: strings
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Modified src/Stabel/Parser.elm from [0e9fb1f9a4] to [387a3a8a3a].

268
269
270
271
272
273
274






275

276
277
278
279
280
281
282
...
307
308
309
310
311
312
313






















314
315
316
317
318
319
320
....
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
textParser =
    Parser.chompWhile (\c -> not <| Set.member c whitespaceChars)
        |> Parser.getChompedString


stringParser : Parser String
stringParser =






    Parser.loop (Just "") stringParserLoop



stringParserLoop : Maybe String -> Parser (Parser.Step (Maybe String) String)
stringParserLoop maybeStr =
    case maybeStr of
        Nothing ->
            -- Work around bug in elm/parser
................................................................................
                , Parser.succeed (Parser.Loop Nothing)
                    |. Parser.end StringNotTerminated
                , Parser.succeed (Parser.Done str)
                    |. Parser.symbol (Token "\n" UnknownError)
                    |> Parser.andThen (\_ -> Parser.problem StringNotTerminated)
                , Parser.succeed ()
                    |. Parser.chompWhile (\c -> c /= '\n' && c /= '"' && c /= '\\')






















                    |> Parser.getChompedString
                    |> Parser.map (\chompedStr -> Parser.Loop <| Just <| str ++ chompedStr)
                ]


genericParser : Parser String
genericParser =
................................................................................
            |. noiseParser
            |= implementationParser
            |. Parser.symbol (Token "}" ExpectedRightCurly)
            |= sourceLocationParser
            |. noiseParser
        , Parser.succeed (\startLoc strContent endLoc -> Parser.Loop (StringLiteral (SourceLocationRange startLoc endLoc) strContent :: nodes))
            |= sourceLocationParser
            |. Parser.symbol (Token "\"" UnknownError)
            |= stringParser
            -- stringParser chomps the final "
            |= sourceLocationParser
            |. noiseParser
        , Parser.succeed (Parser.Done (List.reverse nodes))
        ]








>
>
>
>
>
>
|
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







<







268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
...
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
....
1134
1135
1136
1137
1138
1139
1140

1141
1142
1143
1144
1145
1146
1147
textParser =
    Parser.chompWhile (\c -> not <| Set.member c whitespaceChars)
        |> Parser.getChompedString


stringParser : Parser String
stringParser =
    Parser.oneOf
        [ Parser.succeed identity
            |. Parser.symbol (Token "\"\"\"" UnknownError)
            |= Parser.loop (Just "") multilineStringParserLoop
        , Parser.succeed identity
            |. Parser.symbol (Token "\"" UnknownError)
            |= Parser.loop (Just "") stringParserLoop
        ]


stringParserLoop : Maybe String -> Parser (Parser.Step (Maybe String) String)
stringParserLoop maybeStr =
    case maybeStr of
        Nothing ->
            -- Work around bug in elm/parser
................................................................................
                , Parser.succeed (Parser.Loop Nothing)
                    |. Parser.end StringNotTerminated
                , Parser.succeed (Parser.Done str)
                    |. Parser.symbol (Token "\n" UnknownError)
                    |> Parser.andThen (\_ -> Parser.problem StringNotTerminated)
                , Parser.succeed ()
                    |. Parser.chompWhile (\c -> c /= '\n' && c /= '"' && c /= '\\')
                    |> Parser.getChompedString
                    |> Parser.map (\chompedStr -> Parser.Loop <| Just <| str ++ chompedStr)
                ]


multilineStringParserLoop : Maybe String -> Parser (Parser.Step (Maybe String) String)
multilineStringParserLoop maybeStr =
    case maybeStr of
        Nothing ->
            -- Work around bug in elm/parser
            Parser.problem StringNotTerminated

        Just str ->
            Parser.oneOf
                [ Parser.succeed (Parser.Done str)
                    |. Parser.symbol (Token "\"\"\"" UnknownError)

                -- Couldn't get the below to work in any other way :(
                , Parser.succeed (Parser.Loop Nothing)
                    |. Parser.end StringNotTerminated
                , Parser.succeed ()
                    |. Parser.chompWhile (\c -> c /= '"')
                    |> Parser.getChompedString
                    |> Parser.map (\chompedStr -> Parser.Loop <| Just <| str ++ chompedStr)
                ]


genericParser : Parser String
genericParser =
................................................................................
            |. noiseParser
            |= implementationParser
            |. Parser.symbol (Token "}" ExpectedRightCurly)
            |= sourceLocationParser
            |. noiseParser
        , Parser.succeed (\startLoc strContent endLoc -> Parser.Loop (StringLiteral (SourceLocationRange startLoc endLoc) strContent :: nodes))
            |= sourceLocationParser

            |= stringParser
            -- stringParser chomps the final "
            |= sourceLocationParser
            |. noiseParser
        , Parser.succeed (Parser.Done (List.reverse nodes))
        ]

Modified tests/Test/Parser.elm from [0256c93b49] to [636964fe74].

1210
1211
1212
1213
1214
1215
1216





1217
1218
1219
1220
1221
1222
1223
....
1231
1232
1233
1234
1235
1236
1237








1238




















































                    expectAst source expectedAst
            in
            [ test "Simple case" <|
                \_ ->
                    expectParseString
                        "This is a test"
                        "This is a test"





            , test "Newline escape sequence" <|
                \_ ->
                    expectParseString
                        "This is a test with\\n a newline"
                        "This is a test with\n a newline"
            , test "Tab escape sequence" <|
                \_ ->
................................................................................
                        "This is a test with\\ a backslash"
            , test "Double-quote escape sequence" <|
                \_ ->
                    expectParseString
                        "This is a \\\"test\\\" with a double quote"
                        "This is a \"test\" with a double quote"
            ]








        ]



























































>
>
>
>
>







 







>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
....
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
                    expectAst source expectedAst
            in
            [ test "Simple case" <|
                \_ ->
                    expectParseString
                        "This is a test"
                        "This is a test"
            , test "Empty string" <|
                \_ ->
                    expectParseString
                        ""
                        ""
            , test "Newline escape sequence" <|
                \_ ->
                    expectParseString
                        "This is a test with\\n a newline"
                        "This is a test with\n a newline"
            , test "Tab escape sequence" <|
                \_ ->
................................................................................
                        "This is a test with\\ a backslash"
            , test "Double-quote escape sequence" <|
                \_ ->
                    expectParseString
                        "This is a \\\"test\\\" with a double quote"
                        "This is a \"test\" with a double quote"
            ]
        , describe "Multiline strings" <|
            let
                expectParseString input output =
                    let
                        source =
                            """
                            def: str
                            : """ ++ "\"\"\"" ++ input ++ "\"\"\""

                        expectedAst =
                            { sourceReference = ""
                            , moduleDefinition = ModuleDefinition.Undefined
                            , types = Dict.empty
                            , functions =
                                Dict.fromListBy .name
                                    [ { name = "str"
                                      , typeSignature = AssociatedFunctionSignature.NotProvided
                                      , sourceLocationRange = Nothing
                                      , aliases = Dict.empty
                                      , imports = Dict.empty
                                      , implementation =
                                            SoloImpl
                                                [ AST.StringLiteral emptyRange output
                                                ]
                                      }
                                    ]
                            }
                    in
                    expectAst source expectedAst
            in
            [ test "Simple case" <|
                \_ ->
                    expectParseString
                        "This is a test"
                        "This is a test"
            , test "Empty multiline string" <|
                \_ ->
                    expectParseString
                        ""
                        ""
            , test "There are no escape sequences" <|
                \_ ->
                    expectParseString
                        "This is a test with\\n a newline escape sequence"
                        "This is a test with\\n a newline escape sequence"
            , test "Actual multiline" <|
                \_ ->
                    expectParseString
                        "\n    This is a test\n      with some indentation\n    Let's see how it turns out\n"
                        "This is a test\n  with some indentation\nLet's see how it turns out"
            , test "Above test using Elm's multiline" <|
                \_ ->
                    expectParseString
                        """
                        This is a test
                          with some indentation
                        Let's see how it turns out
                        """
                        "This is a test\n  with some indentation\nLet's see how it turns out"
            ]
        ]

Modified tests/Test/Parser/Errors.elm from [91a4bff812] to [a51137eb07].

304
305
306
307
308
309
310









311
312
313
314
315
316
317
            , test "String never terminates" <|
                \_ ->
                    let
                        source =
                            """
                           def: src
                           : "bad string"""









                    in
                    checkForError stringNotTerminated source
            ]
        ]


checkForError : (Problem -> Bool) -> String -> Expectation







>
>
>
>
>
>
>
>
>







304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
            , test "String never terminates" <|
                \_ ->
                    let
                        source =
                            """
                           def: src
                           : "bad string"""
                    in
                    checkForError stringNotTerminated source
            , test "Multiline string never terminates" <|
                \_ ->
                    let
                        source =
                            """
                           def: src
                           : \"\"\"bad string"""
                    in
                    checkForError stringNotTerminated source
            ]
        ]


checkForError : (Problem -> Bool) -> String -> Expectation