1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
import json
import os
import charset_normalizer
import docx
import markdown
import PyPDF2
import yaml
from bs4 import BeautifulSoup
from pylatexenc.latex2text import LatexNodes2Text
from autogpt import logs
from autogpt.logs import logger
class ParserStrategy:
def read(self, file_path: str) -> str:
raise NotImplementedError
# Basic text file reading
class TXTParser(ParserStrategy):
def read(self, file_path: str) -> str:
charset_match = charset_normalizer.from_path(file_path).best()
logger.debug(f"Reading '{file_path}' with encoding '{charset_match.encoding}'")
return str(charset_match)
# Reading text from binary file using pdf parser
class PDFParser(ParserStrategy):
def read(self, file_path: str) -> str:
parser = PyPDF2.PdfReader(file_path)
text = ""
for page_idx in range(len(parser.pages)):
text += parser.pages[page_idx].extract_text()
return text
# Reading text from binary file using docs parser
class DOCXParser(ParserStrategy):
def read(self, file_path: str) -> str:
doc_file = docx.Document(file_path)
text = ""
for para in doc_file.paragraphs:
text += para.text
return text
# Reading as dictionary and returning string format
class JSONParser(ParserStrategy):
def read(self, file_path: str) -> str:
with open(file_path, "r") as f:
data = json.load(f)
text = str(data)
return text
class XMLParser(ParserStrategy):
def read(self, file_path: str) -> str:
with open(file_path, "r") as f:
soup = BeautifulSoup(f, "xml")
text = soup.get_text()
return text
# Reading as dictionary and returning string format
class YAMLParser(ParserStrategy):
def read(self, file_path: str) -> str:
with open(file_path, "r") as f:
data = yaml.load(f, Loader=yaml.FullLoader)
text = str(data)
return text
class HTMLParser(ParserStrategy):
def read(self, file_path: str) -> str:
with open(file_path, "r") as f:
soup = BeautifulSoup(f, "html.parser")
text = soup.get_text()
return text
class MarkdownParser(ParserStrategy):
def read(self, file_path: str) -> str:
with open(file_path, "r") as f:
html = markdown.markdown(f.read())
text = "".join(BeautifulSoup(html, "html.parser").findAll(string=True))
return text
class LaTeXParser(ParserStrategy):
def read(self, file_path: str) -> str:
with open(file_path, "r") as f:
latex = f.read()
text = LatexNodes2Text().latex_to_text(latex)
return text
class FileContext:
def __init__(self, parser: ParserStrategy, logger: logs.Logger):
self.parser = parser
self.logger = logger
def set_parser(self, parser: ParserStrategy) -> None:
self.logger.debug(f"Setting Context Parser to {parser}")
self.parser = parser
def read_file(self, file_path) -> str:
self.logger.debug(f"Reading file {file_path} with parser {self.parser}")
return self.parser.read(file_path)
extension_to_parser = {
".txt": TXTParser(),
".csv": TXTParser(),
".pdf": PDFParser(),
".docx": DOCXParser(),
".json": JSONParser(),
".xml": XMLParser(),
".yaml": YAMLParser(),
".yml": YAMLParser(),
".html": HTMLParser(),
".htm": HTMLParser(),
".xhtml": HTMLParser(),
".md": MarkdownParser(),
".markdown": MarkdownParser(),
".tex": LaTeXParser(),
}
def is_file_binary_fn(file_path: str):
"""Given a file path load all its content and checks if the null bytes is present
Args:
file_path (_type_): _description_
Returns:
bool: is_binary
"""
with open(file_path, "rb") as f:
file_data = f.read()
if b"\x00" in file_data:
return True
return False
def read_textual_file(file_path: str, logger: logs.Logger) -> str:
if not os.path.isfile(file_path):
raise FileNotFoundError(
f"read_file {file_path} failed: no such file or directory"
)
is_binary = is_file_binary_fn(file_path)
file_extension = os.path.splitext(file_path)[1].lower()
parser = extension_to_parser.get(file_extension)
if not parser:
if is_binary:
raise ValueError(f"Unsupported binary file format: {file_extension}")
# fallback to txt file parser (to support script and code files loading)
parser = TXTParser()
file_context = FileContext(parser, logger)
return file_context.read_file(file_path)
|
