From 703b920286d4380551f678f9819de080cbe0cde6 Mon Sep 17 00:00:00 2001 From: yufansong Date: Sat, 13 Jul 2024 17:58:39 -0700 Subject: [PATCH] gpt swarm init --- .../runtime/plugins/gptswarm_skills/README.md | 0 .../gptswarm_skills/gptswarm_skills.py | 412 ++++++++++++++++++ .../runtime/plugins/gptswarm_skills/setup.sh | 20 + 3 files changed, 432 insertions(+) create mode 100644 opendevin/runtime/plugins/gptswarm_skills/README.md create mode 100644 opendevin/runtime/plugins/gptswarm_skills/gptswarm_skills.py create mode 100644 opendevin/runtime/plugins/gptswarm_skills/setup.sh diff --git a/opendevin/runtime/plugins/gptswarm_skills/README.md b/opendevin/runtime/plugins/gptswarm_skills/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/opendevin/runtime/plugins/gptswarm_skills/gptswarm_skills.py b/opendevin/runtime/plugins/gptswarm_skills/gptswarm_skills.py new file mode 100644 index 0000000000..92cbbb6e70 --- /dev/null +++ b/opendevin/runtime/plugins/gptswarm_skills/gptswarm_skills.py @@ -0,0 +1,412 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from openai import OpenAI +import pdb + +"""INSTALL +pip install openai --upgrade +pip install python-docx +pip install markdown +pip install PyPDF2 +pip install openpyxl +pip install beautifulsoup4 +pip install pylatexenc +pip install python-pptx +pip install xlrd +""" + +import json +import os +import pandas as pd +import charset_normalizer +import docx +import markdown +import PyPDF2 +import openpyxl +import yaml +import zipfile +import subprocess +from pathlib import Path +from abc import ABC, abstractmethod +from typing import Union, Any, Optional +from bs4 import BeautifulSoup +from pylatexenc.latex2text import LatexNodes2Text +from pptx import Presentation + +from swarm.llm import VisualLLMRegistry +from swarm.utils.log import swarmlog, logger +from swarm.utils.globals import Cost + +from dotenv import load_dotenv +load_dotenv() +import aiohttp +import requests +from openai import OpenAI, AsyncOpenAI + +OPENAI_API_KEY=os.getenv("OPENAI_API_KEY") + + +# Refs: https://platform.openai.com/docs/api-reference +# Refs: https://github.com/Significant-Gravitas/AutoGPT/blob/0e332c0c1221857f3ce96490f073c1c88bcbd367/autogpts/autogpt/autogpt/commands/file_operations_utils.py + +class Reader(ABC): + @abstractmethod + def parse(self, file_path: Path) -> str: + """ To be overriden by the descendant class """ + + +class TXTReader(Reader): + def parse(self, file_path: Path) -> str: + content = charset_normalizer.from_path(file_path).best() + #swarmlog("SYS", f"Reading TXT file from {file_path} using encoding '{content.encoding}.'", Cost.instance().value) + logger.info(f"Reading TXT file from {file_path} using encoding '{content.encoding}.'") + return str(content) + +class PDFReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading PDF file from {file_path}.", Cost.instance().value) + logger.info(f"Reading PDF file from {file_path}.") + content = PyPDF2.PdfReader(file_path) + text = "" + for page_idx in range(len(content.pages)): + text += f'Page {page_idx + 1}\n' + content.pages[page_idx].extract_text() + return text + +class DOCXReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading DOCX file from {file_path}.", Cost.instance().value) + logger.info(f"Reading DOCX file from {file_path}.") + content = docx.Document(str(file_path)) + text = "" + for i, para in enumerate(content.paragraphs): + text += f'Page {i + 1}:\n' + para.text + return text + +class JSONReader(Reader): + def parse_file(file_path: Path) -> list: + #swarmlog("SYS", f"Reading JSON file from {file_path}.", Cost.instance().value) + logger.info(f"Reading JSON file from {file_path}.") + try: + with open(file_path, "r") as f: + data = json.load(f) + #text = str(data) + return data#text + except: + return [] + + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading JSON file from {file_path}.", Cost.instance().value) + logger.info(f"Reading JSON file from {file_path}.") + try: + with open(file_path, "r") as f: + data = json.load(f) + text = str(data) + return text + except: + return '' + +class JSONLReader(Reader): + def parse_file(file_path: Path) -> list: + #swarmlog("SYS", f"Reading JSON Lines file from {file_path}.", Cost.instance().value) + logger.info(f"Reading JSON Lines file from {file_path}.") + with open(file_path, "r") as f: + lines = [json.loads(line) for line in f] + #text = '\n'.join([str(line) for line in lines]) + return lines #text + + def parse(file_path: Path) -> str: + #swarmlog("SYS", f"Reading JSON Lines file from {file_path}.", Cost.instance().value) + logger.info(f"Reading JSON Lines file from {file_path}.") + with open(file_path, "r") as f: + lines = [json.loads(line) for line in f] + text = '\n'.join([str(line) for line in lines]) + return text + +class XMLReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading XML file from {file_path}.", Cost.instance().value) + logger.info(f"Reading XML file from {file_path}.") + with open(file_path, "r") as f: + data = BeautifulSoup(f, "xml") + text = data.get_text() + return text + +class YAMLReader(Reader): + def parse(self, file_path: Path, return_str=True) -> Union[str, Any]: + #swarmlog("SYS", f"Reading YAML file from {file_path}.", Cost.instance().value) + logger.info(f"Reading YAML file from {file_path}.") + with open(file_path, "r") as f: + data = yaml.load(f, Loader=yaml.FullLoader) + text = str(data) + if return_str: + return text + else: + return data + +class HTMLReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading HTML file from {file_path}.", Cost.instance().value) + logger.info(f"Reading HTML file from {file_path}.") + with open(file_path, "r") as f: + data = BeautifulSoup(f, "html.parser") + text = data.get_text() + return text + +class MarkdownReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading Markdown file from {file_path}.", Cost.instance().value) + logger.info(f"Reading Markdown file from {file_path}.") + with open(file_path, "r") as f: + data = markdown.markdown(f.read()) + text = "".join(BeautifulSoup(data, "html.parser").findAll(string=True)) + return text + +class LaTexReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading LaTex file from {file_path}.", Cost.instance().value) + logger.info(f"Reading LaTex file from {file_path}.") + with open(file_path, "r") as f: + data = f.read() + text = LatexNodes2Text().latex_to_text(data) + return text + + + +class AudioReader(Reader): + @staticmethod + def parse(file_path: Path) -> str: + #swarmlog("SYS", f"Transcribing audio file from {file_path}.", Cost.instance().value) + logger.info(f"Transcribing audio file from {file_path}.") + client = OpenAI(api_key=OPENAI_API_KEY) + try: + client = OpenAI() + with open(file_path, "rb") as audio_file: + transcript = client.audio.translations.create( + model="whisper-1", + file=audio_file + ) + return transcript.text + except Exception as e: + #swarmlog("ERROR", f"Error transcribing audio file: {e}", Cost.instance().value) + logger.info(f"Error transcribing audio file: {e}") + return "Error transcribing audio file." + +class PPTXReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading PowerPoint file from {file_path}.", Cost.instance().value) + logger.info(f"Reading PowerPoint file from {file_path}.") + try: + pres = Presentation(str(file_path)) + text = [] + for slide_idx, slide in enumerate(pres.slides): + text.append(f"Slide {slide_idx + 1}:\n") + for shape in slide.shapes: + if hasattr(shape, "text"): + text.append(shape.text) + return "\n".join(text) + except Exception as e: + #swarmlog("ERROR", f"Error reading PowerPoint file: {e}", Cost.instance().value) + logger.info(f"Error reading PowerPoint file: {e}") + return "Error reading PowerPoint file." + +class ExcelReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading Excel file from {file_path}.", Cost.instance().value) + logger.info(f"Reading Excel file from {file_path}.") + try: + excel_data = pd.read_excel(file_path, sheet_name=None) + + all_sheets_text = [] + for sheet_name, data in excel_data.items(): + all_sheets_text.append(f"Sheet Name: {sheet_name}\n{data.to_string()}\n") + + return "\n".join(all_sheets_text) + except Exception as e: + #swarmlog("ERROR", f"Error reading Excel file: {e}", Cost.instance().value) + logger.info(f"Error reading Excel file: {e}") + return "Error reading Excel file." + +class XLSXReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Reading XLSX file from {file_path}.", Cost.instance().value) + logger.info(f"Reading XLSX file from {file_path}.") + workbook = openpyxl.load_workbook(file_path, data_only=True) + text = "" + + for sheet in workbook: + text += f"\nSheet: {sheet.title}\n" + for row in sheet.iter_rows(values_only=True): + row_data = [str(cell) if cell is not None else "" for cell in row] + text += "\t".join(row_data) + "\n" + + return text + +class ZipReader(Reader): + def parse(self, file_path: Path) -> str: + #only support files that can be represented as text + #swarmlog("SYS", f"Reading ZIP file from {file_path}.", Cost.instance().value) + logger.info(f"Reading ZIP file from {file_path}.") + try: + file_content = "" + with zipfile.ZipFile(file_path, 'r') as zip_ref: + extract_dir = file_path[:-4] + '/' + zip_ref.extractall(extract_dir) + reader = FileReader() + for file_name in zip_ref.namelist(): + file_content += f'File {file_name}:\n"{reader.read_file(extract_dir + file_name)}"\n' + return file_content + + except zipfile.BadZipFile: + #swarmlog("ERROR", "Invalid ZIP file.", Cost.instance().value) + logger.info("Invalid ZIP file.") + + except Exception as e: + #swarmlog("ERROR", f"Error reading ZIP file: {e}", Cost.instance().value) + logger.info(f"Error reading ZIP file: {e}") + + +class PythonReader(Reader): + def parse(self, file_path: Path) -> str: + #swarmlog("SYS", f"Executing and reading Python file from {file_path}.", Cost.instance().value) + logger.info(f"Executing and reading Python file from {file_path}.") + execution_result = "" + error = "" + file_content = "" + try: + completed_process = subprocess.run(["python", file_path], capture_output=True, text=True, check=True) + execution_result = "Output:\n" + completed_process.stdout + except subprocess.CalledProcessError as e: + error = "Error:\n" + e.stderr + except Exception as e: + #swarmlog("ERROR", f"Error executing Python file: {e}", Cost.instance().value) + logger.info(f"Error executing Python file: {e}") + + try: + with open(file_path, "r") as file: + file_content = "\nFile Content:\n" + file.read() + except Exception as e: + #swarmlog("ERROR", f"Error reading Python file: {e}", Cost.instance().value) + logger.info(f"Error reading Python file: {e}") + return file_content, execution_result, error + + +class IMGReader(Reader): + def parse(self, file_path: Path, task: str = "Describe this image as detail as possible." ) -> str: + #swarmlog("SYS", f"Reading image file from {file_path}.", Cost.instance().value) + logger.info(f"Reading image file from {file_path}.") + runner = VisualLLMRegistry.get() + answer = runner.gen(task, file_path) + return answer + +class VideoReader(Reader): + def parse(self, file_path: Path, task: str = "Describe this image as detail as possible.", frame_interval: int = 30, used_audio: bool = True) -> list: + #swarmlog("SYS", f"Processing video file from {file_path} with frame interval {frame_interval}.", Cost.instance().value) + logger.info(f"Processing video file from {file_path} with frame interval {frame_interval}.") + runner = VisualLLMRegistry.get() + answer = runner.gen_video(task, file_path, frame_interval) + + if used_audio: + audio_content = AudioReader.parse(file_path) + + return answer + "The audio includes:\n" + audio_content + + +# Support 41 kinds of files. +READER_MAP = { + ".png": IMGReader(), + ".jpg": IMGReader(), + ".jpeg": IMGReader(), + ".gif": IMGReader(), + ".bmp": IMGReader(), + ".tiff": IMGReader(), + ".tif": IMGReader(), + ".webp": IMGReader(), + ".mp3": AudioReader(), + ".m4a": AudioReader(), + ".wav": AudioReader(), + ".MOV": VideoReader(), + ".mp4": VideoReader(), + ".mov": VideoReader(), + ".avi": VideoReader(), + ".mpg": VideoReader(), + ".mpeg": VideoReader(), + ".wmv": VideoReader(), + ".flv": VideoReader(), + ".webm": VideoReader(), + ".zip": ZipReader(), + ".pptx": PPTXReader(), + ".xlsx": ExcelReader(), + ".xls": ExcelReader(), + ".txt": TXTReader(), + ".csv": TXTReader(), + ".pdf": PDFReader(), + ".docx": DOCXReader(), + ".json": JSONReader(), + ".jsonld": JSONReader(), + ".jsonl": JSONLReader(), + ".xml": XMLReader(), + ".yaml": YAMLReader(), + ".yml": YAMLReader(), + ".html": HTMLReader(), + ".htm": HTMLReader(), + ".xhtml": HTMLReader(), + ".md": MarkdownReader(), + ".markdown": MarkdownReader(), + ".tex": LaTexReader(), + ".py": PythonReader(), + ".pdb": TXTReader(), +} + +class FileReader: + def set_reader(self, suffix) -> None: + self.reader = READER_MAP[suffix] + #swarmlog("SYS", f"Setting Reader to {type(self.reader).__name__}", Cost.instance().value) + logger.info(f"Setting Reader to {type(self.reader).__name__}") + + def read_file(self, file_path: Path, task="describe the file") -> str: + suffix = '.' + file_path.split(".")[-1] + self.set_reader(suffix) + if isinstance(self.reader, IMGReader) or isinstance(self.reader, VideoReader): + file_content = self.reader.parse(file_path, task) + else: + file_content = self.reader.parse(file_path) + #swarmlog("SYS", f"Reading file {file_path} using {type(self.reader).__name__}", Cost.instance().value) + logger.info(f"Reading file {file_path} using {type(self.reader).__name__}") + return file_content + + +class GeneralReader: + def __init__(self): + self.file_reader = FileReader() + self.name = "General File Reader" + self.description = """A general file reader support to formats: 'py', 'java', 'cpp', 'c', 'js', + 'css', 'html', 'htm', 'xml', 'txt', 'jsonl', 'csv', 'json', + 'jsonld', 'jsonl', 'yaml', 'yml', 'xlsx', 'xls', 'jpg', 'png', + 'jpeg', 'gif', 'bmp', 'mp3', 'wav', 'ogg', 'mp4', 'avi', 'mkv', + 'mov', 'pdf', 'doc', 'docx', 'ppt', 'pptx', 'md', 'markdown', + 'tex', 'zip', 'tar', 'gz', '7z', 'rar'. + """ + + def read(self, task, file): + + files_content = "" + file_content = self.file_reader.read_file(file, task) + suffix = file.split(".")[-1] + + if suffix in ['py', 'java', 'cpp', 'c', 'js', 'css', 'html', 'htm', 'xml']: + files_content += f'\nThe {suffix} file contains:\n---\n{file_content[0]}' + if file_content[1] != '': + files_content += f'\nExecution result:\n{file_content[1]}' + if file_content[2] != '': + files_content += f'\nExecution error message:\n{file_content[2]}' + files_content += '\n---' + + elif suffix in ['txt', 'jsonl', 'csv', 'json', 'jsonld', 'jsonl', 'yaml', 'yml', + 'xlsx', 'xls', 'jpg', 'png', 'jpeg', 'gif', 'bmp', 'mp3', 'wav', + 'ogg', 'mp4', 'avi', 'mkv', 'mov', 'pdf', 'doc', 'docx', 'ppt', + 'pptx', 'md', 'markdown', 'tex', 'zip', 'tar', 'gz', '7z', 'rar']: + files_content += f'\nThe {suffix} file contains:\n---\n{file_content}\n---' + + return files_content + \ No newline at end of file diff --git a/opendevin/runtime/plugins/gptswarm_skills/setup.sh b/opendevin/runtime/plugins/gptswarm_skills/setup.sh new file mode 100644 index 0000000000..cfdd9db587 --- /dev/null +++ b/opendevin/runtime/plugins/gptswarm_skills/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +OPENDEVIN_PYTHON_INTERPRETER=/opendevin/miniforge3/bin/python +# check if OPENDEVIN_PYTHON_INTERPRETER exists and it is usable +if [ -z "$OPENDEVIN_PYTHON_INTERPRETER" ] || [ ! -x "$OPENDEVIN_PYTHON_INTERPRETER" ]; then + echo "OPENDEVIN_PYTHON_INTERPRETER is not usable. Please pull the latest Docker image!" + exit 1 +fi + +# add gptswarm_skills to PATH +echo 'export PATH=/opendevin/plugins/gptswarm_skills:$PATH' >> ~/.bashrc + +# add gptswarm_skills to PYTHONPATH +echo 'export PYTHONPATH=/opendevin/plugins/gptswarm_skills:$PYTHONPATH' >> ~/.bashrc + +source ~/.bashrc + +$OPENDEVIN_PYTHON_INTERPRETER -m pip install flake8 python-docx PyPDF2 python-pptx pylatexenc openai opencv-python