Stabel

Check-in [0e154d19e7]
Login
Overview
Comment:Hex-encoded integer literals.
Timelines: family | ancestors | descendants | both | integer-syntax-improvements
Files: files | file ages | folders
SHA3-256: 0e154d19e7b5440a7684dbf0cd60abcf5c6c82817753ec66994d36ac4f7c3cb9
User & Date: robin.hansen on 2021-09-16 14:02:28
Other Links: branch diff | manifest | tags
Context
2021-09-16
14:28
Bit-encoded integer literals. Closed-Leaf check-in: 317c5ea0b3 user: robin.hansen tags: integer-syntax-improvements
14:02
Hex-encoded integer literals. check-in: 0e154d19e7 user: robin.hansen tags: integer-syntax-improvements
13:05
Check that integers are within range of 32-bit signed. check-in: db0d5f7a64 user: robin.hansen tags: integer-syntax-improvements
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Modified elm.json from [6ed21b8cff] to [1fc5ced1fe].

5
6
7
8
9
10
11

12
13
14
15
16
17
18
..
19
20
21
22
23
24
25
26
27
28
29
30
31
32
    ],
    "elm-version": "0.19.1",
    "dependencies": {
        "direct": {
            "elm/core": "1.0.5",
            "elm/json": "1.1.3",
            "elm/parser": "1.1.0",

            "elm-community/dict-extra": "2.4.0",
            "elm-community/list-extra": "8.3.0",
            "elm-community/result-extra": "2.4.0",
            "zwilias/elm-utf-tools": "2.0.1"
        },
        "indirect": {}
    },
................................................................................
    "test-dependencies": {
        "direct": {
            "elm-community/string-extra": "4.0.1",
            "elm-explorations/test": "1.2.2"
        },
        "indirect": {
            "elm/html": "1.0.0",
            "elm/random": "1.0.0",
            "elm/regex": "1.0.0",
            "elm/time": "1.0.0",
            "elm/virtual-dom": "1.0.2"
        }
    }
}







>







 







<






5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
..
20
21
22
23
24
25
26

27
28
29
30
31
32
    ],
    "elm-version": "0.19.1",
    "dependencies": {
        "direct": {
            "elm/core": "1.0.5",
            "elm/json": "1.1.3",
            "elm/parser": "1.1.0",
            "elm/random": "1.0.0",
            "elm-community/dict-extra": "2.4.0",
            "elm-community/list-extra": "8.3.0",
            "elm-community/result-extra": "2.4.0",
            "zwilias/elm-utf-tools": "2.0.1"
        },
        "indirect": {}
    },
................................................................................
    "test-dependencies": {
        "direct": {
            "elm-community/string-extra": "4.0.1",
            "elm-explorations/test": "1.2.2"
        },
        "indirect": {
            "elm/html": "1.0.0",

            "elm/regex": "1.0.0",
            "elm/time": "1.0.0",
            "elm/virtual-dom": "1.0.2"
        }
    }
}

Modified src/Stabel/Parser.elm from [f0e2701f0c] to [00441a56f7].

6
7
8
9
10
11
12

13
14

15
16
17
18
19
20
21
...
134
135
136
137
138
139
140
141




142
143
144
145
146















147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164

165
166









































167
168
169
170
171
172
173
    , TypeDefinition
    , TypeDefinitionMembers(..)
    , TypeMatch(..)
    , TypeMatchValue(..)
    , run
    )


import Dict exposing (Dict)
import Dict.Extra as Dict

import Parser.Advanced as Parser exposing ((|.), (|=), Token(..))
import Random
import Set exposing (Set)
import Stabel.Parser.AssociatedFunctionSignature as AssociatedFunctionSignature exposing (AssociatedFunctionSignature)
import Stabel.Parser.ModuleDefinition as ModuleDefinition exposing (ModuleDefinition)
import Stabel.Parser.Problem as Problem exposing (Context, Problem(..))
import Stabel.Parser.SourceLocation exposing (SourceLocation, SourceLocationRange)
................................................................................
    Set.fromList
        [ ' '
        , '\n'
        , '\u{000D}'
        , '\t'
        ]






{-| The builtin int parser has a bug where it commits when it comes across an 'e'
-}
intParser : Parser Int
intParser =
    Parser.inContext Problem.IntegerLiteral















        (Parser.succeed Tuple.pair
            |= Parser.variable
                { start = Char.isDigit
                , inner = \c -> Char.isDigit c || c == '_'
                , reserved = Set.empty
                , expecting = ExpectedInt
                }
            |= Parser.oneOf
                [ Parser.succeed True
                    |. Parser.symbol (Token "-" UnknownError)
                , Parser.succeed False
                ]
            |. Parser.oneOf
                [ Parser.chompIf (\c -> Set.member c whitespaceChars) ExpectedWhitespace
                , Parser.succeed ()
                    |. Parser.end ExpectedEndOfFile
                ]
            |> Parser.andThen intParserHelper

        )











































intParserHelper : ( String, Bool ) -> Parser Int
intParserHelper ( text, isNegative ) =
    let
        digits =
            String.replace "_" "" text
    in







>


>







 








>
>
>
>





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>


>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
...
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
    , TypeDefinition
    , TypeDefinitionMembers(..)
    , TypeMatch(..)
    , TypeMatchValue(..)
    , run
    )

import Bitwise
import Dict exposing (Dict)
import Dict.Extra as Dict
import List.Extra as List
import Parser.Advanced as Parser exposing ((|.), (|=), Token(..))
import Random
import Set exposing (Set)
import Stabel.Parser.AssociatedFunctionSignature as AssociatedFunctionSignature exposing (AssociatedFunctionSignature)
import Stabel.Parser.ModuleDefinition as ModuleDefinition exposing (ModuleDefinition)
import Stabel.Parser.Problem as Problem exposing (Context, Problem(..))
import Stabel.Parser.SourceLocation exposing (SourceLocation, SourceLocationRange)
................................................................................
    Set.fromList
        [ ' '
        , '\n'
        , '\u{000D}'
        , '\t'
        ]



-- Int parsing


{-| The builtin int parser has a bug where it commits when it comes across an 'e'
-}
intParser : Parser Int
intParser =
    Parser.inContext Problem.IntegerLiteral
        (Parser.oneOf
            [ Parser.succeed identity
                |. Parser.symbol (Token "0x" UnknownError)
                |= Parser.variable
                    { start = isHexDigit
                    , inner = isHexDigit
                    , reserved = Set.empty
                    , expecting = ExpectedHexInt
                    }
                |. Parser.oneOf
                    [ Parser.chompIf (\c -> Set.member c whitespaceChars) ExpectedWhitespace
                    , Parser.succeed ()
                        |. Parser.end ExpectedEndOfFile
                    ]
                |> Parser.andThen intHexParserHelper
            , Parser.succeed Tuple.pair
                |= Parser.variable
                    { start = Char.isDigit
                    , inner = \c -> Char.isDigit c || c == '_'
                    , reserved = Set.empty
                    , expecting = ExpectedInt
                    }
                |= Parser.oneOf
                    [ Parser.succeed True
                        |. Parser.symbol (Token "-" UnknownError)
                    , Parser.succeed False
                    ]
                |. Parser.oneOf
                    [ Parser.chompIf (\c -> Set.member c whitespaceChars) ExpectedWhitespace
                    , Parser.succeed ()
                        |. Parser.end ExpectedEndOfFile
                    ]
                |> Parser.andThen intParserHelper
            ]
        )



-- Hex integers


isHexDigit : Char -> Bool
isHexDigit char =
    List.member (Char.toUpper char) hexDigits


hexDigits : List Char
hexDigits =
    [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' ]


intHexParserHelper : String -> Parser Int
intHexParserHelper text =
    if String.length text > 8 then
        Parser.problem IntegerHexOutOfBounds

    else
        String.foldl hexCharFolder 0 text
            |> Parser.succeed


hexCharFolder : Char -> Int -> Int
hexCharFolder char num =
    num
        |> Bitwise.shiftLeftBy 4
        |> Bitwise.or (hexCharToNum char)


hexCharToNum : Char -> Int
hexCharToNum char =
    List.elemIndex (Char.toUpper char) hexDigits
        |> Maybe.withDefault -1



-- Base 10 integers


intParserHelper : ( String, Bool ) -> Parser Int
intParserHelper ( text, isNegative ) =
    let
        digits =
            String.replace "_" "" text
    in

Modified src/Stabel/Parser/Problem.elm from [fffc0c5e37] to [6a8b426c68].

30
31
32
33
34
35
36

37
38
39
40
41
42
43
..
57
58
59
60
61
62
63

64
65
66
67
68
69
70
...
201
202
203
204
205
206
207



208
209
210
211
212
213
214
...
304
305
306
307
308
309
310



    | IntegerLiteral
    | ArrayLiteral
    | StringLiteral


type Problem
    = ExpectedInt

    | ExpectedSymbol
    | ExpectedMetadata
    | ExpectedGeneric
    | ExpectedType
    | UnknownError
    | ExpectedForwardSlash
    | ExpectedWhitespace
................................................................................
    | ModuleIsEmpty
    | BadDefinition String
    | UnknownEscapeSequence String
    | StringNotTerminated
    | IntegerBadLeadingZero
    | IntegerTrailingUnderscore
    | IntegerOutOfBounds



toString : String -> String -> DeadEnd Context Problem -> String
toString sourceRef source deadEnd =
    let
        contextExplination =
            contextStackExplination deadEnd.contextStack
................................................................................

problemToString : String -> Problem -> String
problemToString source problem =
    case problem of
        ExpectedInt ->
            "Expected to find an integer"




        ExpectedSymbol ->
            "Expected to find a symbol"

        ExpectedMetadata ->
            "Expected to find a keyword"

        ExpectedGeneric ->
................................................................................
            "Integers cannot start with 0."

        IntegerTrailingUnderscore ->
            "Integers cannot end with an underscore."

        IntegerOutOfBounds ->
            "Integers must fit within a signed 32-bit number."










>







 







>







 







>
>
>







 







>
>
>
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
..
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
...
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
...
309
310
311
312
313
314
315
316
317
318
    | IntegerLiteral
    | ArrayLiteral
    | StringLiteral


type Problem
    = ExpectedInt
    | ExpectedHexInt
    | ExpectedSymbol
    | ExpectedMetadata
    | ExpectedGeneric
    | ExpectedType
    | UnknownError
    | ExpectedForwardSlash
    | ExpectedWhitespace
................................................................................
    | ModuleIsEmpty
    | BadDefinition String
    | UnknownEscapeSequence String
    | StringNotTerminated
    | IntegerBadLeadingZero
    | IntegerTrailingUnderscore
    | IntegerOutOfBounds
    | IntegerHexOutOfBounds


toString : String -> String -> DeadEnd Context Problem -> String
toString sourceRef source deadEnd =
    let
        contextExplination =
            contextStackExplination deadEnd.contextStack
................................................................................

problemToString : String -> Problem -> String
problemToString source problem =
    case problem of
        ExpectedInt ->
            "Expected to find an integer"

        ExpectedHexInt ->
            "Expected to find an hex-encoded integer"

        ExpectedSymbol ->
            "Expected to find a symbol"

        ExpectedMetadata ->
            "Expected to find a keyword"

        ExpectedGeneric ->
................................................................................
            "Integers cannot start with 0."

        IntegerTrailingUnderscore ->
            "Integers cannot end with an underscore."

        IntegerOutOfBounds ->
            "Integers must fit within a signed 32-bit number."

        IntegerHexOutOfBounds ->
            "Hex-encoded integers can contain at most 8 characters (4 bytes, 32-bits)."

Modified tests/Test/Parser.elm from [78a2b93463] to [66acdf25a7].

1534
1535
1536
1537
1538
1539
1540




1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
                    expectParseInt (String.fromInt num) num
            , fuzz negativeIntFuzzer "Negative ints" <|
                \num ->
                    expectParseInt (String.fromInt (num * -1) ++ "-") num
            , test "can contain underscores as seperators" <|
                \_ ->
                    expectParseInt "10_000" 10000




            ]
        ]


positiveIntFuzzer : Fuzzer Int
positiveIntFuzzer =
    Fuzz.intRange 0 Random.maxInt


negativeIntFuzzer : Fuzzer Int
negativeIntFuzzer =
    Fuzz.intRange Random.minInt -1







>
>
>
>












1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
                    expectParseInt (String.fromInt num) num
            , fuzz negativeIntFuzzer "Negative ints" <|
                \num ->
                    expectParseInt (String.fromInt (num * -1) ++ "-") num
            , test "can contain underscores as seperators" <|
                \_ ->
                    expectParseInt "10_000" 10000
            , test "can be in Hex format" <|
                \_ -> expectParseInt "0xFA0" 0x0FA0
            , test "can be in Hex format (second example)" <|
                \_ -> expectParseInt "0xe3cc" 0xE3CC
            ]
        ]


positiveIntFuzzer : Fuzzer Int
positiveIntFuzzer =
    Fuzz.intRange 0 Random.maxInt


negativeIntFuzzer : Fuzzer Int
negativeIntFuzzer =
    Fuzz.intRange Random.minInt -1

Modified tests/Test/Parser/Errors.elm from [528443d252] to [5a3cb2c60e].

367
368
369
370
371
372
373








































374
375
376
377
378
379
380
                        source =
                            """
                            def: src
                            : 3_000_000_000-
                            """
                    in
                    checkForError ((==) IntegerOutOfBounds) source








































            ]
        ]


checkForError : (Problem -> Bool) -> String -> Expectation
checkForError fn source =
    case compile source of







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
                        source =
                            """
                            def: src
                            : 3_000_000_000-
                            """
                    in
                    checkForError ((==) IntegerOutOfBounds) source
            , test "Hex numbers cannot be negative" <|
                \_ ->
                    let
                        source =
                            """
                            def: src
                            : 0xAA00FF-
                            """
                    in
                    checkForError ((==) ExpectedWhitespace) source
            , test "Hex numbers cannot use underscores" <|
                \_ ->
                    let
                        source =
                            """
                            def: src
                            : 0xAA_00FF
                            """
                    in
                    checkForError ((==) ExpectedWhitespace) source
            , test "Hex numbers must contain at least one member" <|
                \_ ->
                    let
                        source =
                            """
                            def: src
                            : 0x
                            """
                    in
                    checkForError ((==) ExpectedHexInt) source
            , test "Hex numbers cannot contain more than 8 characters (4 bytes, 32 bits)" <|
                \_ ->
                    let
                        source =
                            """
                            def: src
                            : 0xAABBCCDDE
                            """
                    in
                    checkForError ((==) IntegerHexOutOfBounds) source
            ]
        ]


checkForError : (Problem -> Bool) -> String -> Expectation
checkForError fn source =
    case compile source of