1 import sys 2 from contextlib import contextmanager 3 from typing import ( 4 Annotated, 5 AnyStr, 6 NamedTuple, 7 Literal, 8 NewType, 9 Optional, 10 Protocol, 11 TypeGuard, 12 Union, 13 TypedDict, 14 ) 15 from functools import partial 16 from IPython.core.guarded_eval import ( 17 EvaluationContext, 18 GuardRejection, 19 guarded_eval, 20 _unbind_method, 21 ) 22 from IPython.testing import decorators as dec 23 import pytest 24 25 26 if sys.version_info < (3, 11): 27 from typing_extensions import Self, LiteralString 28 else: 29 from typing import Self, LiteralString 30 31 if sys.version_info < (3, 12): 32 from typing_extensions import TypeAliasType 33 else: 34 from typing import TypeAliasType 35 36 37 def create_context(evaluation: str, **kwargs): 38 return EvaluationContext(locals=kwargs, globals={}, evaluation=evaluation) 39 40 41 forbidden = partial(create_context, "forbidden") 42 minimal = partial(create_context, "minimal") 43 limited = partial(create_context, "limited") 44 unsafe = partial(create_context, "unsafe") 45 dangerous = partial(create_context, "dangerous") 46 47 LIMITED_OR_HIGHER = [limited, unsafe, dangerous] 48 MINIMAL_OR_HIGHER = [minimal, *LIMITED_OR_HIGHER] 49 50 51 @contextmanager 52 def module_not_installed(module: str): 53 import sys 54 55 try: 56 to_restore = sys.modules[module] 57 del sys.modules[module] 58 except KeyError: 59 to_restore = None 60 try: 61 yield 62 finally: 63 sys.modules[module] = to_restore 64 65 66 def test_external_not_installed(): 67 """ 68 Because attribute check requires checking if object is not of allowed 69 external type, this tests logic for absence of external module. 70 """ 71 72 class Custom: 73 def __init__(self): 74 self.test = 1 75 76 def __getattr__(self, key): 77 return key 78 79 with module_not_installed("pandas"): 80 context = limited(x=Custom()) 81 with pytest.raises(GuardRejection): 82 guarded_eval("x.test", context) 83 84 85 @dec.skip_without("pandas") 86 def test_external_changed_api(monkeypatch): 87 """Check that the execution rejects if external API changed paths""" 88 import pandas as pd 89 90 series = pd.Series([1], index=["a"]) 91 92 with monkeypatch.context() as m: 93 m.delattr(pd, "Series") 94 context = limited(data=series) 95 with pytest.raises(GuardRejection): 96 guarded_eval("data.iloc[0]", context) 97 98 99 @dec.skip_without("pandas") 100 def test_pandas_series_iloc(): 101 import pandas as pd 102 103 series = pd.Series([1], index=["a"]) 104 context = limited(data=series) 105 assert guarded_eval("data.iloc[0]", context) == 1 106 107 108 def test_rejects_custom_properties(): 109 class BadProperty: 110 @property 111 def iloc(self): 112 return [None] 113 114 series = BadProperty() 115 context = limited(data=series) 116 117 with pytest.raises(GuardRejection): 118 guarded_eval("data.iloc[0]", context) 119 120 121 @dec.skip_without("pandas") 122 def test_accepts_non_overriden_properties(): 123 import pandas as pd 124 125 class GoodProperty(pd.Series): 126 pass 127 128 series = GoodProperty([1], index=["a"]) 129 context = limited(data=series) 130 131 assert guarded_eval("data.iloc[0]", context) == 1 132 133 134 @dec.skip_without("pandas") 135 def test_pandas_series(): 136 import pandas as pd 137 138 context = limited(data=pd.Series([1], index=["a"])) 139 assert guarded_eval('data["a"]', context) == 1 140 with pytest.raises(KeyError): 141 guarded_eval('data["c"]', context) 142 143 144 @dec.skip_without("pandas") 145 def test_pandas_bad_series(): 146 import pandas as pd 147 148 class BadItemSeries(pd.Series): 149 def __getitem__(self, key): 150 return "CUSTOM_ITEM" 151 152 class BadAttrSeries(pd.Series): 153 def __getattr__(self, key): 154 return "CUSTOM_ATTR" 155 156 bad_series = BadItemSeries([1], index=["a"]) 157 context = limited(data=bad_series) 158 159 with pytest.raises(GuardRejection): 160 guarded_eval('data["a"]', context) 161 with pytest.raises(GuardRejection): 162 guarded_eval('data["c"]', context) 163 164 # note: here result is a bit unexpected because 165 # pandas `__getattr__` calls `__getitem__`; 166 # FIXME - special case to handle it? 167 assert guarded_eval("data.a", context) == "CUSTOM_ITEM" 168 169 context = unsafe(data=bad_series) 170 assert guarded_eval('data["a"]', context) == "CUSTOM_ITEM" 171 172 bad_attr_series = BadAttrSeries([1], index=["a"]) 173 context = limited(data=bad_attr_series) 174 assert guarded_eval('data["a"]', context) == 1 175 with pytest.raises(GuardRejection): 176 guarded_eval("data.a", context) 177 178 179 @dec.skip_without("pandas") 180 def test_pandas_dataframe_loc(): 181 import pandas as pd 182 from pandas.testing import assert_series_equal 183 184 data = pd.DataFrame([{"a": 1}]) 185 context = limited(data=data) 186 assert_series_equal(guarded_eval('data.loc[:, "a"]', context), data["a"]) 187 188 189 def test_named_tuple(): 190 class GoodNamedTuple(NamedTuple): 191 a: str 192 pass 193 194 class BadNamedTuple(NamedTuple): 195 a: str 196 197 def __getitem__(self, key): 198 return None 199 200 good = GoodNamedTuple(a="x") 201 bad = BadNamedTuple(a="x") 202 203 context = limited(data=good) 204 assert guarded_eval("data[0]", context) == "x" 205 206 context = limited(data=bad) 207 with pytest.raises(GuardRejection): 208 guarded_eval("data[0]", context) 209 210 211 def test_dict(): 212 context = limited(data={"a": 1, "b": {"x": 2}, ("x", "y"): 3}) 213 assert guarded_eval('data["a"]', context) == 1 214 assert guarded_eval('data["b"]', context) == {"x": 2} 215 assert guarded_eval('data["b"]["x"]', context) == 2 216 assert guarded_eval('data["x", "y"]', context) == 3 217 218 assert guarded_eval("data.keys", context) 219 220 221 def test_set(): 222 context = limited(data={"a", "b"}) 223 assert guarded_eval("data.difference", context) 224 225 226 def test_list(): 227 context = limited(data=[1, 2, 3]) 228 assert guarded_eval("data[1]", context) == 2 229 assert guarded_eval("data.copy", context) 230 231 232 def test_dict_literal(): 233 context = limited() 234 assert guarded_eval("{}", context) == {} 235 assert guarded_eval('{"a": 1}', context) == {"a": 1} 236 237 238 def test_list_literal(): 239 context = limited() 240 assert guarded_eval("[]", context) == [] 241 assert guarded_eval('[1, "a"]', context) == [1, "a"] 242 243 244 def test_set_literal(): 245 context = limited() 246 assert guarded_eval("set()", context) == set() 247 assert guarded_eval('{"a"}', context) == {"a"} 248 249 250 def test_evaluates_if_expression(): 251 context = limited() 252 assert guarded_eval("2 if True else 3", context) == 2 253 assert guarded_eval("4 if False else 5", context) == 5 254 255 256 def test_object(): 257 obj = object() 258 context = limited(obj=obj) 259 assert guarded_eval("obj.__dir__", context) == obj.__dir__ 260 261 262 @pytest.mark.parametrize( 263 "code,expected", 264 [ 265 ["int.numerator", int.numerator], 266 ["float.is_integer", float.is_integer], 267 ["complex.real", complex.real], 268 ], 269 ) 270 def test_number_attributes(code, expected): 271 assert guarded_eval(code, limited()) == expected 272 273 274 def test_method_descriptor(): 275 context = limited() 276 assert guarded_eval("list.copy.__name__", context) == "copy" 277 278 279 class HeapType: 280 pass 281 282 283 class CallCreatesHeapType: 284 def __call__(self) -> HeapType: 285 return HeapType() 286 287 288 class CallCreatesBuiltin: 289 def __call__(self) -> frozenset: 290 return frozenset() 291 292 293 class HasStaticMethod: 294 @staticmethod 295 def static_method() -> HeapType: 296 return HeapType() 297 298 299 class InitReturnsFrozenset: 300 def __new__(self) -> frozenset: # type:ignore[misc] 301 return frozenset() 302 303 304 class StringAnnotation: 305 def heap(self) -> "HeapType": 306 return HeapType() 307 308 def copy(self) -> "StringAnnotation": 309 return StringAnnotation() 310 311 312 CustomIntType = NewType("CustomIntType", int) 313 CustomHeapType = NewType("CustomHeapType", HeapType) 314 IntTypeAlias = TypeAliasType("IntTypeAlias", int) 315 HeapTypeAlias = TypeAliasType("HeapTypeAlias", HeapType) 316 317 318 class TestProtocol(Protocol): 319 def test_method(self) -> bool: 320 pass 321 322 323 class TestProtocolImplementer(TestProtocol): 324 def test_method(self) -> bool: 325 return True 326 327 328 class Movie(TypedDict): 329 name: str 330 year: int 331 332 333 class SpecialTyping: 334 def custom_int_type(self) -> CustomIntType: 335 return CustomIntType(1) 336 337 def custom_heap_type(self) -> CustomHeapType: 338 return CustomHeapType(HeapType()) 339 340 # TODO: remove type:ignore comment once mypy 341 # supports explicit calls to `TypeAliasType`, see: 342 # https://github.com/python/mypy/issues/16614 343 def int_type_alias(self) -> IntTypeAlias: # type:ignore[valid-type] 344 return 1 345 346 def heap_type_alias(self) -> HeapTypeAlias: # type:ignore[valid-type] 347 return 1 348 349 def literal(self) -> Literal[False]: 350 return False 351 352 def literal_string(self) -> LiteralString: 353 return "test" 354 355 def self(self) -> Self: 356 return self 357 358 def any_str(self, x: AnyStr) -> AnyStr: 359 return x 360 361 def annotated(self) -> Annotated[float, "positive number"]: 362 return 1 363 364 def annotated_self(self) -> Annotated[Self, "self with metadata"]: 365 self._metadata = "test" 366 return self 367 368 def int_type_guard(self, x) -> TypeGuard[int]: 369 return isinstance(x, int) 370 371 def optional_float(self) -> Optional[float]: 372 return 1.0 373 374 def union_str_and_int(self) -> Union[str, int]: 375 return "" 376 377 def protocol(self) -> TestProtocol: 378 return TestProtocolImplementer() 379 380 def typed_dict(self) -> Movie: 381 return {"name": "The Matrix", "year": 1999} 382 383 384 @pytest.mark.parametrize( 385 "data,code,expected,equality", 386 [ 387 [[1, 2, 3], "data.index(2)", 1, True], 388 [{"a": 1}, "data.keys().isdisjoint({})", True, True], 389 [StringAnnotation(), "data.heap()", HeapType, False], 390 [StringAnnotation(), "data.copy()", StringAnnotation, False], 391 # test cases for `__call__` 392 [CallCreatesHeapType(), "data()", HeapType, False], 393 [CallCreatesBuiltin(), "data()", frozenset, False], 394 # Test cases for `__init__` 395 [HeapType, "data()", HeapType, False], 396 [InitReturnsFrozenset, "data()", frozenset, False], 397 [HeapType(), "data.__class__()", HeapType, False], 398 # supported special cases for typing 399 [SpecialTyping(), "data.custom_int_type()", int, False], 400 [SpecialTyping(), "data.custom_heap_type()", HeapType, False], 401 [SpecialTyping(), "data.int_type_alias()", int, False], 402 [SpecialTyping(), "data.heap_type_alias()", HeapType, False], 403 [SpecialTyping(), "data.self()", SpecialTyping, False], 404 [SpecialTyping(), "data.literal()", False, True], 405 [SpecialTyping(), "data.literal_string()", str, False], 406 [SpecialTyping(), "data.any_str('a')", str, False], 407 [SpecialTyping(), "data.any_str(b'a')", bytes, False], 408 [SpecialTyping(), "data.annotated()", float, False], 409 [SpecialTyping(), "data.annotated_self()", SpecialTyping, False], 410 [SpecialTyping(), "data.int_type_guard()", int, False], 411 # test cases for static methods 412 [HasStaticMethod, "data.static_method()", HeapType, False], 413 ], 414 ) 415 def test_evaluates_calls(data, code, expected, equality): 416 context = limited(data=data, HeapType=HeapType, StringAnnotation=StringAnnotation) 417 value = guarded_eval(code, context) 418 if equality: 419 assert value == expected 420 else: 421 assert isinstance(value, expected) 422 423 424 @pytest.mark.parametrize( 425 "data,code,expected_attributes", 426 [ 427 [SpecialTyping(), "data.optional_float()", ["is_integer"]], 428 [ 429 SpecialTyping(), 430 "data.union_str_and_int()", 431 ["capitalize", "as_integer_ratio"], 432 ], 433 [SpecialTyping(), "data.protocol()", ["test_method"]], 434 [SpecialTyping(), "data.typed_dict()", ["keys", "values", "items"]], 435 ], 436 ) 437 def test_mocks_attributes_of_call_results(data, code, expected_attributes): 438 context = limited(data=data, HeapType=HeapType, StringAnnotation=StringAnnotation) 439 result = guarded_eval(code, context) 440 for attr in expected_attributes: 441 assert hasattr(result, attr) 442 assert attr in dir(result) 443 444 445 @pytest.mark.parametrize( 446 "data,code,expected_items", 447 [ 448 [SpecialTyping(), "data.typed_dict()", {"year": int, "name": str}], 449 ], 450 ) 451 def test_mocks_items_of_call_results(data, code, expected_items): 452 context = limited(data=data, HeapType=HeapType, StringAnnotation=StringAnnotation) 453 result = guarded_eval(code, context) 454 ipython_keys = result._ipython_key_completions_() 455 for key, value in expected_items.items(): 456 assert isinstance(result[key], value) 457 assert key in ipython_keys 458 459 460 @pytest.mark.parametrize( 461 "data,bad", 462 [ 463 [[1, 2, 3], "data.append(4)"], 464 [{"a": 1}, "data.update()"], 465 ], 466 ) 467 def test_rejects_calls_with_side_effects(data, bad): 468 context = limited(data=data) 469 470 with pytest.raises(GuardRejection): 471 guarded_eval(bad, context) 472 473 474 @pytest.mark.parametrize( 475 "code,expected", 476 [ 477 ["(1\n+\n1)", 2], 478 ["list(range(10))[-1:]", [9]], 479 ["list(range(20))[3:-2:3]", [3, 6, 9, 12, 15]], 480 ], 481 ) 482 @pytest.mark.parametrize("context", LIMITED_OR_HIGHER) 483 def test_evaluates_complex_cases(code, expected, context): 484 assert guarded_eval(code, context()) == expected 485 486 487 @pytest.mark.parametrize( 488 "code,expected", 489 [ 490 ["1", 1], 491 ["1.0", 1.0], 492 ["0xdeedbeef", 0xDEEDBEEF], 493 ["True", True], 494 ["None", None], 495 ["{}", {}], 496 ["[]", []], 497 ], 498 ) 499 @pytest.mark.parametrize("context", MINIMAL_OR_HIGHER) 500 def test_evaluates_literals(code, expected, context): 501 assert guarded_eval(code, context()) == expected 502 503 504 @pytest.mark.parametrize( 505 "code,expected", 506 [ 507 ["-5", -5], 508 ["+5", +5], 509 ["~5", -6], 510 ], 511 ) 512 @pytest.mark.parametrize("context", LIMITED_OR_HIGHER) 513 def test_evaluates_unary_operations(code, expected, context): 514 assert guarded_eval(code, context()) == expected 515 516 517 @pytest.mark.parametrize( 518 "code,expected", 519 [ 520 ["1 + 1", 2], 521 ["3 - 1", 2], 522 ["2 * 3", 6], 523 ["5 // 2", 2], 524 ["5 / 2", 2.5], 525 ["5**2", 25], 526 ["2 >> 1", 1], 527 ["2 << 1", 4], 528 ["1 | 2", 3], 529 ["1 & 1", 1], 530 ["1 & 2", 0], 531 ], 532 ) 533 @pytest.mark.parametrize("context", LIMITED_OR_HIGHER) 534 def test_evaluates_binary_operations(code, expected, context): 535 assert guarded_eval(code, context()) == expected 536 537 538 @pytest.mark.parametrize( 539 "code,expected", 540 [ 541 ["2 > 1", True], 542 ["2 < 1", False], 543 ["2 <= 1", False], 544 ["2 <= 2", True], 545 ["1 >= 2", False], 546 ["2 >= 2", True], 547 ["2 == 2", True], 548 ["1 == 2", False], 549 ["1 != 2", True], 550 ["1 != 1", False], 551 ["1 < 4 < 3", False], 552 ["(1 < 4) < 3", True], 553 ["4 > 3 > 2 > 1", True], 554 ["4 > 3 > 2 > 9", False], 555 ["1 < 2 < 3 < 4", True], 556 ["9 < 2 < 3 < 4", False], 557 ["1 < 2 > 1 > 0 > -1 < 1", True], 558 ["1 in [1] in [[1]]", True], 559 ["1 in [1] in [[2]]", False], 560 ["1 in [1]", True], 561 ["0 in [1]", False], 562 ["1 not in [1]", False], 563 ["0 not in [1]", True], 564 ["True is True", True], 565 ["False is False", True], 566 ["True is False", False], 567 ["True is not True", False], 568 ["False is not True", True], 569 ], 570 ) 571 @pytest.mark.parametrize("context", LIMITED_OR_HIGHER) 572 def test_evaluates_comparisons(code, expected, context): 573 assert guarded_eval(code, context()) == expected 574 575 576 def test_guards_comparisons(): 577 class GoodEq(int): 578 pass 579 580 class BadEq(int): 581 def __eq__(self, other): 582 assert False 583 584 context = limited(bad=BadEq(1), good=GoodEq(1)) 585 586 with pytest.raises(GuardRejection): 587 guarded_eval("bad == 1", context) 588 589 with pytest.raises(GuardRejection): 590 guarded_eval("bad != 1", context) 591 592 with pytest.raises(GuardRejection): 593 guarded_eval("1 == bad", context) 594 595 with pytest.raises(GuardRejection): 596 guarded_eval("1 != bad", context) 597 598 assert guarded_eval("good == 1", context) is True 599 assert guarded_eval("good != 1", context) is False 600 assert guarded_eval("1 == good", context) is True 601 assert guarded_eval("1 != good", context) is False 602 603 604 def test_guards_unary_operations(): 605 class GoodOp(int): 606 pass 607 608 class BadOpInv(int): 609 def __inv__(self, other): 610 assert False 611 612 class BadOpInverse(int): 613 def __inv__(self, other): 614 assert False 615 616 context = limited(good=GoodOp(1), bad1=BadOpInv(1), bad2=BadOpInverse(1)) 617 618 with pytest.raises(GuardRejection): 619 guarded_eval("~bad1", context) 620 621 with pytest.raises(GuardRejection): 622 guarded_eval("~bad2", context) 623 624 625 def test_guards_binary_operations(): 626 class GoodOp(int): 627 pass 628 629 class BadOp(int): 630 def __add__(self, other): 631 assert False 632 633 context = limited(good=GoodOp(1), bad=BadOp(1)) 634 635 with pytest.raises(GuardRejection): 636 guarded_eval("1 + bad", context) 637 638 with pytest.raises(GuardRejection): 639 guarded_eval("bad + 1", context) 640 641 assert guarded_eval("good + 1", context) == 2 642 assert guarded_eval("1 + good", context) == 2 643 644 645 def test_guards_attributes(): 646 class GoodAttr(float): 647 pass 648 649 class BadAttr1(float): 650 def __getattr__(self, key): 651 assert False 652 653 class BadAttr2(float): 654 def __getattribute__(self, key): 655 assert False 656 657 context = limited(good=GoodAttr(0.5), bad1=BadAttr1(0.5), bad2=BadAttr2(0.5)) 658 659 with pytest.raises(GuardRejection): 660 guarded_eval("bad1.as_integer_ratio", context) 661 662 with pytest.raises(GuardRejection): 663 guarded_eval("bad2.as_integer_ratio", context) 664 665 assert guarded_eval("good.as_integer_ratio()", context) == (1, 2) 666 667 668 @pytest.mark.parametrize("context", MINIMAL_OR_HIGHER) 669 def test_access_builtins(context): 670 assert guarded_eval("round", context()) == round 671 672 673 def test_access_builtins_fails(): 674 context = limited() 675 with pytest.raises(NameError): 676 guarded_eval("this_is_not_builtin", context) 677 678 679 def test_rejects_forbidden(): 680 context = forbidden() 681 with pytest.raises(GuardRejection): 682 guarded_eval("1", context) 683 684 685 def test_guards_locals_and_globals(): 686 context = EvaluationContext( 687 locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="minimal" 688 ) 689 690 with pytest.raises(GuardRejection): 691 guarded_eval("local_a", context) 692 693 with pytest.raises(GuardRejection): 694 guarded_eval("global_b", context) 695 696 697 def test_access_locals_and_globals(): 698 context = EvaluationContext( 699 locals={"local_a": "a"}, globals={"global_b": "b"}, evaluation="limited" 700 ) 701 assert guarded_eval("local_a", context) == "a" 702 assert guarded_eval("global_b", context) == "b" 703 704 705 @pytest.mark.parametrize( 706 "code", 707 ["def func(): pass", "class C: pass", "x = 1", "x += 1", "del x", "import ast"], 708 ) 709 @pytest.mark.parametrize("context", [minimal(), limited(), unsafe()]) 710 def test_rejects_side_effect_syntax(code, context): 711 with pytest.raises(SyntaxError): 712 guarded_eval(code, context) 713 714 715 def test_subscript(): 716 context = EvaluationContext( 717 locals={}, globals={}, evaluation="limited", in_subscript=True 718 ) 719 empty_slice = slice(None, None, None) 720 assert guarded_eval("", context) == tuple() 721 assert guarded_eval(":", context) == empty_slice 722 assert guarded_eval("1:2:3", context) == slice(1, 2, 3) 723 assert guarded_eval(':, "a"', context) == (empty_slice, "a") 724 725 726 def test_unbind_method(): 727 class X(list): 728 def index(self, k): 729 return "CUSTOM" 730 731 x = X() 732 assert _unbind_method(x.index) is X.index 733 assert _unbind_method([].index) is list.index 734 assert _unbind_method(list.index) is None 735 736 737 def test_assumption_instance_attr_do_not_matter(): 738 """This is semi-specified in Python documentation. 739 740 However, since the specification says 'not guaranteed 741 to work' rather than 'is forbidden to work', future 742 versions could invalidate this assumptions. This test 743 is meant to catch such a change if it ever comes true. 744 """ 745 746 class T: 747 def __getitem__(self, k): 748 return "a" 749 750 def __getattr__(self, k): 751 return "a" 752 753 def f(self): 754 return "b" 755 756 t = T() 757 t.__getitem__ = f 758 t.__getattr__ = f 759 assert t[1] == "a" 760 assert t[1] == "a" 761 762 763 def test_assumption_named_tuples_share_getitem(): 764 """Check assumption on named tuples sharing __getitem__""" 765 from typing import NamedTuple 766 767 class A(NamedTuple): 768 pass 769 770 class B(NamedTuple): 771 pass 772 773 assert A.__getitem__ == B.__getitem__ 774 775 776 @dec.skip_without("numpy") 777 def test_module_access(): 778 import numpy 779 780 context = limited(numpy=numpy) 781 assert guarded_eval("numpy.linalg.norm", context) == numpy.linalg.norm 782 783 context = minimal(numpy=numpy) 784 with pytest.raises(GuardRejection): 785 guarded_eval("np.linalg.norm", context) 786