1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
|
import pytest
from pytest import raises
from autogpt.url_utils.validators import validate_url
"""
Code Analysis
Objective:
The objective of the 'validate_url' function is to validate URLs for any command that requires a URL as an argument. It checks if the URL is valid using a basic check, urllib check, and local file check. If the URL fails any of the validation tests, it raises a ValueError.
Inputs:
- func: A callable function that takes in any number of arguments and returns any type of output.
Flow:
- The 'validate_url' function takes in a callable function as an argument.
- It defines a wrapper function that takes in a URL and any number of arguments and keyword arguments.
- The wrapper function first checks if the URL starts with "http://" or "https://". If not, it raises a ValueError with the message "Invalid URL format".
- It then checks if the URL is valid using the 'is_valid_url' function. If not, it raises a ValueError with the message "Missing Scheme or Network location".
- It then checks if the URL is a local file using the 'check_local_file_access' function. If it is, it raises a ValueError with the message "Access to local files is restricted".
- If the URL passes all the validation tests, it sanitizes the URL using the 'sanitize_url' function and calls the original function with the sanitized URL and any other arguments and keyword arguments.
- The wrapper function returns the result of the original function.
Outputs:
- The 'validate_url' function returns the wrapper function that takes in a URL and any number of arguments and keyword arguments and returns the result of the original function.
Additional aspects:
- The 'validate_url' function uses the 'functools.wraps' decorator to preserve the original function's metadata, such as its name, docstring, and annotations.
- The 'validate_url' function uses the 'urlparse' function from the 'urllib.parse' module to parse the URL and extract its components.
- The 'validate_url' function uses the 'urljoin' function from the 'requests.compat' module to join the sanitized URL components back into a URL string.
"""
@validate_url
def dummy_method(url):
return url
successful_test_data = (
("https://google.com/search?query=abc"),
("https://google.com/search?query=abc&p=123"),
("http://google.com/"),
("http://a.lot.of.domain.net/param1/param2"),
)
@pytest.mark.parametrize("url", successful_test_data)
def test_url_validation_succeeds(url):
assert dummy_method(url) == url
@pytest.mark.parametrize(
"url,expected_error",
[
("htt://example.com", "Invalid URL format"),
("httppp://example.com", "Invalid URL format"),
(" https://example.com", "Invalid URL format"),
("http://?query=q", "Missing Scheme or Network location"),
],
)
def test_url_validation_fails_invalid_url(url, expected_error):
with raises(ValueError, match=expected_error):
dummy_method(url)
local_file = (
("http://localhost"),
("https://localhost/"),
("http://2130706433"),
("https://2130706433"),
("http://127.0.0.1/"),
)
@pytest.mark.parametrize("url", local_file)
def test_url_validation_fails_local_path(url):
with raises(ValueError, match="Access to local files is restricted"):
dummy_method(url)
class TestValidateUrl:
# Tests that the function successfully validates a valid URL with http:// or https:// prefix.
def test_happy_path_valid_url(self):
"""Test that the function successfully validates a valid URL with http:// or https:// prefix"""
@validate_url
def test_func(url):
return url
assert test_func("https://www.google.com") == "https://www.google.com"
assert test_func("http://www.google.com") == "http://www.google.com"
# Tests that the function successfully validates a valid URL with additional path, parameters, and query string.
def test_general_behavior_additional_path_parameters_query_string(self):
"""Test that the function successfully validates a valid URL with additional path, parameters, and query string"""
@validate_url
def test_func(url):
return url
assert (
test_func("https://www.google.com/search?q=python")
== "https://www.google.com/search?q=python"
)
# Tests that the function raises a ValueError if the URL is missing scheme or network location.
def test_edge_case_missing_scheme_or_network_location(self):
"""Test that the function raises a ValueError if the URL is missing scheme or network location"""
@validate_url
def test_func(url):
return url
with pytest.raises(ValueError):
test_func("www.google.com")
# Tests that the function raises a ValueError if the URL has local file access.
def test_edge_case_local_file_access(self):
"""Test that the function raises a ValueError if the URL has local file access"""
@validate_url
def test_func(url):
return url
with pytest.raises(ValueError):
test_func("file:///etc/passwd")
# Tests that the function sanitizes the URL by removing any unnecessary components.
def test_general_behavior_sanitizes_url(self):
"""Test that the function sanitizes the URL by removing any unnecessary components"""
@validate_url
def test_func(url):
return url
assert (
test_func("https://www.google.com/search?q=python#top")
== "https://www.google.com/search?q=python"
)
# Tests that the function raises a ValueError if the URL has an invalid format (e.g. missing slashes).
def test_general_behavior_invalid_url_format(self):
"""Test that the function raises a ValueError if the URL has an invalid format (e.g. missing slashes)"""
@validate_url
def test_func(url):
return url
with pytest.raises(ValueError):
test_func("https:www.google.com")
# Tests that the function can handle URLs that contain unusual but valid characters.
def test_url_with_special_chars(self):
url = "https://example.com/path%20with%20spaces"
assert dummy_method(url) == url
# Tests that the function raises a ValueError if the URL is over 2000 characters.
def test_extremely_long_url(self):
url = "http://example.com/" + "a" * 2000
with raises(ValueError, match="URL is too long"):
dummy_method(url)
# Tests that the function can handle internationalized URLs, which contain non-ASCII characters.
def test_internationalized_url(self):
url = "http://例子.测试"
assert dummy_method(url) == url
|
