mirror of
https://github.com/OpenHands/OpenHands.git
synced 2025-12-26 05:48:36 +08:00
gpt swarm init
This commit is contained in:
yufansong
parent
46edb4b15b
commit
703b920286
0
opendevin/runtime/plugins/gptswarm_skills/README.md
Normal file
0
opendevin/runtime/plugins/gptswarm_skills/README.md
Normal file
412
opendevin/runtime/plugins/gptswarm_skills/gptswarm_skills.py
Normal file
412
opendevin/runtime/plugins/gptswarm_skills/gptswarm_skills.py
Normal file
@ -0,0 +1,412 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from openai import OpenAI
|
||||
import pdb
|
||||
|
||||
"""INSTALL
|
||||
pip install openai --upgrade
|
||||
pip install python-docx
|
||||
pip install markdown
|
||||
pip install PyPDF2
|
||||
pip install openpyxl
|
||||
pip install beautifulsoup4
|
||||
pip install pylatexenc
|
||||
pip install python-pptx
|
||||
pip install xlrd
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import pandas as pd
|
||||
import charset_normalizer
|
||||
import docx
|
||||
import markdown
|
||||
import PyPDF2
|
||||
import openpyxl
|
||||
import yaml
|
||||
import zipfile
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Union, Any, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
from pylatexenc.latex2text import LatexNodes2Text
|
||||
from pptx import Presentation
|
||||
|
||||
from swarm.llm import VisualLLMRegistry
|
||||
from swarm.utils.log import swarmlog, logger
|
||||
from swarm.utils.globals import Cost
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
import aiohttp
|
||||
import requests
|
||||
from openai import OpenAI, AsyncOpenAI
|
||||
|
||||
OPENAI_API_KEY=os.getenv("OPENAI_API_KEY")
|
||||
|
||||
|
||||
# Refs: https://platform.openai.com/docs/api-reference
|
||||
# Refs: https://github.com/Significant-Gravitas/AutoGPT/blob/0e332c0c1221857f3ce96490f073c1c88bcbd367/autogpts/autogpt/autogpt/commands/file_operations_utils.py
|
||||
|
||||
class Reader(ABC):
|
||||
@abstractmethod
|
||||
def parse(self, file_path: Path) -> str:
|
||||
""" To be overriden by the descendant class """
|
||||
|
||||
|
||||
class TXTReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
content = charset_normalizer.from_path(file_path).best()
|
||||
#swarmlog("SYS", f"Reading TXT file from {file_path} using encoding '{content.encoding}.'", Cost.instance().value)
|
||||
logger.info(f"Reading TXT file from {file_path} using encoding '{content.encoding}.'")
|
||||
return str(content)
|
||||
|
||||
class PDFReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading PDF file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading PDF file from {file_path}.")
|
||||
content = PyPDF2.PdfReader(file_path)
|
||||
text = ""
|
||||
for page_idx in range(len(content.pages)):
|
||||
text += f'Page {page_idx + 1}\n' + content.pages[page_idx].extract_text()
|
||||
return text
|
||||
|
||||
class DOCXReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading DOCX file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading DOCX file from {file_path}.")
|
||||
content = docx.Document(str(file_path))
|
||||
text = ""
|
||||
for i, para in enumerate(content.paragraphs):
|
||||
text += f'Page {i + 1}:\n' + para.text
|
||||
return text
|
||||
|
||||
class JSONReader(Reader):
|
||||
def parse_file(file_path: Path) -> list:
|
||||
#swarmlog("SYS", f"Reading JSON file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading JSON file from {file_path}.")
|
||||
try:
|
||||
with open(file_path, "r") as f:
|
||||
data = json.load(f)
|
||||
#text = str(data)
|
||||
return data#text
|
||||
except:
|
||||
return []
|
||||
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading JSON file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading JSON file from {file_path}.")
|
||||
try:
|
||||
with open(file_path, "r") as f:
|
||||
data = json.load(f)
|
||||
text = str(data)
|
||||
return text
|
||||
except:
|
||||
return ''
|
||||
|
||||
class JSONLReader(Reader):
|
||||
def parse_file(file_path: Path) -> list:
|
||||
#swarmlog("SYS", f"Reading JSON Lines file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading JSON Lines file from {file_path}.")
|
||||
with open(file_path, "r") as f:
|
||||
lines = [json.loads(line) for line in f]
|
||||
#text = '\n'.join([str(line) for line in lines])
|
||||
return lines #text
|
||||
|
||||
def parse(file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading JSON Lines file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading JSON Lines file from {file_path}.")
|
||||
with open(file_path, "r") as f:
|
||||
lines = [json.loads(line) for line in f]
|
||||
text = '\n'.join([str(line) for line in lines])
|
||||
return text
|
||||
|
||||
class XMLReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading XML file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading XML file from {file_path}.")
|
||||
with open(file_path, "r") as f:
|
||||
data = BeautifulSoup(f, "xml")
|
||||
text = data.get_text()
|
||||
return text
|
||||
|
||||
class YAMLReader(Reader):
|
||||
def parse(self, file_path: Path, return_str=True) -> Union[str, Any]:
|
||||
#swarmlog("SYS", f"Reading YAML file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading YAML file from {file_path}.")
|
||||
with open(file_path, "r") as f:
|
||||
data = yaml.load(f, Loader=yaml.FullLoader)
|
||||
text = str(data)
|
||||
if return_str:
|
||||
return text
|
||||
else:
|
||||
return data
|
||||
|
||||
class HTMLReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading HTML file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading HTML file from {file_path}.")
|
||||
with open(file_path, "r") as f:
|
||||
data = BeautifulSoup(f, "html.parser")
|
||||
text = data.get_text()
|
||||
return text
|
||||
|
||||
class MarkdownReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading Markdown file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading Markdown file from {file_path}.")
|
||||
with open(file_path, "r") as f:
|
||||
data = markdown.markdown(f.read())
|
||||
text = "".join(BeautifulSoup(data, "html.parser").findAll(string=True))
|
||||
return text
|
||||
|
||||
class LaTexReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading LaTex file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading LaTex file from {file_path}.")
|
||||
with open(file_path, "r") as f:
|
||||
data = f.read()
|
||||
text = LatexNodes2Text().latex_to_text(data)
|
||||
return text
|
||||
|
||||
|
||||
|
||||
class AudioReader(Reader):
|
||||
@staticmethod
|
||||
def parse(file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Transcribing audio file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Transcribing audio file from {file_path}.")
|
||||
client = OpenAI(api_key=OPENAI_API_KEY)
|
||||
try:
|
||||
client = OpenAI()
|
||||
with open(file_path, "rb") as audio_file:
|
||||
transcript = client.audio.translations.create(
|
||||
model="whisper-1",
|
||||
file=audio_file
|
||||
)
|
||||
return transcript.text
|
||||
except Exception as e:
|
||||
#swarmlog("ERROR", f"Error transcribing audio file: {e}", Cost.instance().value)
|
||||
logger.info(f"Error transcribing audio file: {e}")
|
||||
return "Error transcribing audio file."
|
||||
|
||||
class PPTXReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading PowerPoint file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading PowerPoint file from {file_path}.")
|
||||
try:
|
||||
pres = Presentation(str(file_path))
|
||||
text = []
|
||||
for slide_idx, slide in enumerate(pres.slides):
|
||||
text.append(f"Slide {slide_idx + 1}:\n")
|
||||
for shape in slide.shapes:
|
||||
if hasattr(shape, "text"):
|
||||
text.append(shape.text)
|
||||
return "\n".join(text)
|
||||
except Exception as e:
|
||||
#swarmlog("ERROR", f"Error reading PowerPoint file: {e}", Cost.instance().value)
|
||||
logger.info(f"Error reading PowerPoint file: {e}")
|
||||
return "Error reading PowerPoint file."
|
||||
|
||||
class ExcelReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading Excel file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading Excel file from {file_path}.")
|
||||
try:
|
||||
excel_data = pd.read_excel(file_path, sheet_name=None)
|
||||
|
||||
all_sheets_text = []
|
||||
for sheet_name, data in excel_data.items():
|
||||
all_sheets_text.append(f"Sheet Name: {sheet_name}\n{data.to_string()}\n")
|
||||
|
||||
return "\n".join(all_sheets_text)
|
||||
except Exception as e:
|
||||
#swarmlog("ERROR", f"Error reading Excel file: {e}", Cost.instance().value)
|
||||
logger.info(f"Error reading Excel file: {e}")
|
||||
return "Error reading Excel file."
|
||||
|
||||
class XLSXReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Reading XLSX file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading XLSX file from {file_path}.")
|
||||
workbook = openpyxl.load_workbook(file_path, data_only=True)
|
||||
text = ""
|
||||
|
||||
for sheet in workbook:
|
||||
text += f"\nSheet: {sheet.title}\n"
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
row_data = [str(cell) if cell is not None else "" for cell in row]
|
||||
text += "\t".join(row_data) + "\n"
|
||||
|
||||
return text
|
||||
|
||||
class ZipReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#only support files that can be represented as text
|
||||
#swarmlog("SYS", f"Reading ZIP file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading ZIP file from {file_path}.")
|
||||
try:
|
||||
file_content = ""
|
||||
with zipfile.ZipFile(file_path, 'r') as zip_ref:
|
||||
extract_dir = file_path[:-4] + '/'
|
||||
zip_ref.extractall(extract_dir)
|
||||
reader = FileReader()
|
||||
for file_name in zip_ref.namelist():
|
||||
file_content += f'File {file_name}:\n"{reader.read_file(extract_dir + file_name)}"\n'
|
||||
return file_content
|
||||
|
||||
except zipfile.BadZipFile:
|
||||
#swarmlog("ERROR", "Invalid ZIP file.", Cost.instance().value)
|
||||
logger.info("Invalid ZIP file.")
|
||||
|
||||
except Exception as e:
|
||||
#swarmlog("ERROR", f"Error reading ZIP file: {e}", Cost.instance().value)
|
||||
logger.info(f"Error reading ZIP file: {e}")
|
||||
|
||||
|
||||
class PythonReader(Reader):
|
||||
def parse(self, file_path: Path) -> str:
|
||||
#swarmlog("SYS", f"Executing and reading Python file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Executing and reading Python file from {file_path}.")
|
||||
execution_result = ""
|
||||
error = ""
|
||||
file_content = ""
|
||||
try:
|
||||
completed_process = subprocess.run(["python", file_path], capture_output=True, text=True, check=True)
|
||||
execution_result = "Output:\n" + completed_process.stdout
|
||||
except subprocess.CalledProcessError as e:
|
||||
error = "Error:\n" + e.stderr
|
||||
except Exception as e:
|
||||
#swarmlog("ERROR", f"Error executing Python file: {e}", Cost.instance().value)
|
||||
logger.info(f"Error executing Python file: {e}")
|
||||
|
||||
try:
|
||||
with open(file_path, "r") as file:
|
||||
file_content = "\nFile Content:\n" + file.read()
|
||||
except Exception as e:
|
||||
#swarmlog("ERROR", f"Error reading Python file: {e}", Cost.instance().value)
|
||||
logger.info(f"Error reading Python file: {e}")
|
||||
return file_content, execution_result, error
|
||||
|
||||
|
||||
class IMGReader(Reader):
|
||||
def parse(self, file_path: Path, task: str = "Describe this image as detail as possible." ) -> str:
|
||||
#swarmlog("SYS", f"Reading image file from {file_path}.", Cost.instance().value)
|
||||
logger.info(f"Reading image file from {file_path}.")
|
||||
runner = VisualLLMRegistry.get()
|
||||
answer = runner.gen(task, file_path)
|
||||
return answer
|
||||
|
||||
class VideoReader(Reader):
|
||||
def parse(self, file_path: Path, task: str = "Describe this image as detail as possible.", frame_interval: int = 30, used_audio: bool = True) -> list:
|
||||
#swarmlog("SYS", f"Processing video file from {file_path} with frame interval {frame_interval}.", Cost.instance().value)
|
||||
logger.info(f"Processing video file from {file_path} with frame interval {frame_interval}.")
|
||||
runner = VisualLLMRegistry.get()
|
||||
answer = runner.gen_video(task, file_path, frame_interval)
|
||||
|
||||
if used_audio:
|
||||
audio_content = AudioReader.parse(file_path)
|
||||
|
||||
return answer + "The audio includes:\n" + audio_content
|
||||
|
||||
|
||||
# Support 41 kinds of files.
|
||||
READER_MAP = {
|
||||
".png": IMGReader(),
|
||||
".jpg": IMGReader(),
|
||||
".jpeg": IMGReader(),
|
||||
".gif": IMGReader(),
|
||||
".bmp": IMGReader(),
|
||||
".tiff": IMGReader(),
|
||||
".tif": IMGReader(),
|
||||
".webp": IMGReader(),
|
||||
".mp3": AudioReader(),
|
||||
".m4a": AudioReader(),
|
||||
".wav": AudioReader(),
|
||||
".MOV": VideoReader(),
|
||||
".mp4": VideoReader(),
|
||||
".mov": VideoReader(),
|
||||
".avi": VideoReader(),
|
||||
".mpg": VideoReader(),
|
||||
".mpeg": VideoReader(),
|
||||
".wmv": VideoReader(),
|
||||
".flv": VideoReader(),
|
||||
".webm": VideoReader(),
|
||||
".zip": ZipReader(),
|
||||
".pptx": PPTXReader(),
|
||||
".xlsx": ExcelReader(),
|
||||
".xls": ExcelReader(),
|
||||
".txt": TXTReader(),
|
||||
".csv": TXTReader(),
|
||||
".pdf": PDFReader(),
|
||||
".docx": DOCXReader(),
|
||||
".json": JSONReader(),
|
||||
".jsonld": JSONReader(),
|
||||
".jsonl": JSONLReader(),
|
||||
".xml": XMLReader(),
|
||||
".yaml": YAMLReader(),
|
||||
".yml": YAMLReader(),
|
||||
".html": HTMLReader(),
|
||||
".htm": HTMLReader(),
|
||||
".xhtml": HTMLReader(),
|
||||
".md": MarkdownReader(),
|
||||
".markdown": MarkdownReader(),
|
||||
".tex": LaTexReader(),
|
||||
".py": PythonReader(),
|
||||
".pdb": TXTReader(),
|
||||
}
|
||||
|
||||
class FileReader:
|
||||
def set_reader(self, suffix) -> None:
|
||||
self.reader = READER_MAP[suffix]
|
||||
#swarmlog("SYS", f"Setting Reader to {type(self.reader).__name__}", Cost.instance().value)
|
||||
logger.info(f"Setting Reader to {type(self.reader).__name__}")
|
||||
|
||||
def read_file(self, file_path: Path, task="describe the file") -> str:
|
||||
suffix = '.' + file_path.split(".")[-1]
|
||||
self.set_reader(suffix)
|
||||
if isinstance(self.reader, IMGReader) or isinstance(self.reader, VideoReader):
|
||||
file_content = self.reader.parse(file_path, task)
|
||||
else:
|
||||
file_content = self.reader.parse(file_path)
|
||||
#swarmlog("SYS", f"Reading file {file_path} using {type(self.reader).__name__}", Cost.instance().value)
|
||||
logger.info(f"Reading file {file_path} using {type(self.reader).__name__}")
|
||||
return file_content
|
||||
|
||||
|
||||
class GeneralReader:
|
||||
def __init__(self):
|
||||
self.file_reader = FileReader()
|
||||
self.name = "General File Reader"
|
||||
self.description = """A general file reader support to formats: 'py', 'java', 'cpp', 'c', 'js',
|
||||
'css', 'html', 'htm', 'xml', 'txt', 'jsonl', 'csv', 'json',
|
||||
'jsonld', 'jsonl', 'yaml', 'yml', 'xlsx', 'xls', 'jpg', 'png',
|
||||
'jpeg', 'gif', 'bmp', 'mp3', 'wav', 'ogg', 'mp4', 'avi', 'mkv',
|
||||
'mov', 'pdf', 'doc', 'docx', 'ppt', 'pptx', 'md', 'markdown',
|
||||
'tex', 'zip', 'tar', 'gz', '7z', 'rar'.
|
||||
"""
|
||||
|
||||
def read(self, task, file):
|
||||
|
||||
files_content = ""
|
||||
file_content = self.file_reader.read_file(file, task)
|
||||
suffix = file.split(".")[-1]
|
||||
|
||||
if suffix in ['py', 'java', 'cpp', 'c', 'js', 'css', 'html', 'htm', 'xml']:
|
||||
files_content += f'\nThe {suffix} file contains:\n---\n{file_content[0]}'
|
||||
if file_content[1] != '':
|
||||
files_content += f'\nExecution result:\n{file_content[1]}'
|
||||
if file_content[2] != '':
|
||||
files_content += f'\nExecution error message:\n{file_content[2]}'
|
||||
files_content += '\n---'
|
||||
|
||||
elif suffix in ['txt', 'jsonl', 'csv', 'json', 'jsonld', 'jsonl', 'yaml', 'yml',
|
||||
'xlsx', 'xls', 'jpg', 'png', 'jpeg', 'gif', 'bmp', 'mp3', 'wav',
|
||||
'ogg', 'mp4', 'avi', 'mkv', 'mov', 'pdf', 'doc', 'docx', 'ppt',
|
||||
'pptx', 'md', 'markdown', 'tex', 'zip', 'tar', 'gz', '7z', 'rar']:
|
||||
files_content += f'\nThe {suffix} file contains:\n---\n{file_content}\n---'
|
||||
|
||||
return files_content
|
||||
|
||||
20
opendevin/runtime/plugins/gptswarm_skills/setup.sh
Normal file
20
opendevin/runtime/plugins/gptswarm_skills/setup.sh
Normal file
@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
OPENDEVIN_PYTHON_INTERPRETER=/opendevin/miniforge3/bin/python
|
||||
# check if OPENDEVIN_PYTHON_INTERPRETER exists and it is usable
|
||||
if [ -z "$OPENDEVIN_PYTHON_INTERPRETER" ] || [ ! -x "$OPENDEVIN_PYTHON_INTERPRETER" ]; then
|
||||
echo "OPENDEVIN_PYTHON_INTERPRETER is not usable. Please pull the latest Docker image!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# add gptswarm_skills to PATH
|
||||
echo 'export PATH=/opendevin/plugins/gptswarm_skills:$PATH' >> ~/.bashrc
|
||||
|
||||
# add gptswarm_skills to PYTHONPATH
|
||||
echo 'export PYTHONPATH=/opendevin/plugins/gptswarm_skills:$PYTHONPATH' >> ~/.bashrc
|
||||
|
||||
source ~/.bashrc
|
||||
|
||||
$OPENDEVIN_PYTHON_INTERPRETER -m pip install flake8 python-docx PyPDF2 python-pptx pylatexenc openai opencv-python
|
||||
Loading…
x
Reference in New Issue
Block a user