Merge pull request 'Using subcommands instead of arguments' (#2) from subcommands into main
Reviewed-on: https://kaizenkodo.no/gitea/kaizenkodo/ChatMastermind/pulls/2
This commit is contained in:
commit
e8eba0b755
80
README.md
80
README.md
@ -29,65 +29,101 @@ pip install .
|
|||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
The `cmm` script has global options, a list of commands, and options per command:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmm [-h] [-p PRINT | -q QUESTION | -D | -d | -l] [-c CONFIG] [-m MAX_TOKENS] [-T TEMPERATURE] [-M MODEL] [-n NUMBER] [-t [TAGS [TAGS ...]]] [-e [EXTAGS [EXTAGS ...]]] [-o [OTAGS [OTAGS ...]]] [-a] [-w] [-W]
|
cmm [global options] command [command options]
|
||||||
```
|
```
|
||||||
|
|
||||||
### Arguments
|
### Global Options
|
||||||
|
|
||||||
- `-p`, `--print`: YAML file to print.
|
|
||||||
- `-q`, `--question`: Question to ask.
|
|
||||||
- `-D`, `--chat-dump`: Print chat history as a Python structure.
|
|
||||||
- `-d`, `--chat`: Print chat history as readable text.
|
|
||||||
- `-a`, `--match-all-tags`: All given tags must match when selecting chat history entries.
|
|
||||||
- `-w`, `--with-tags`: Print chat history with tags.
|
|
||||||
- `-W`, `--with-tags`: Print chat history with filenames.
|
|
||||||
- `-l`, `--list-tags`: List all tags and their frequency.
|
|
||||||
- `-c`, `--config`: Config file name (defaults to `.config.yaml`).
|
- `-c`, `--config`: Config file name (defaults to `.config.yaml`).
|
||||||
|
|
||||||
|
### Commands
|
||||||
|
|
||||||
|
- `ask`: Ask a question.
|
||||||
|
- `hist`: Print chat history.
|
||||||
|
- `tag`: Manage tags.
|
||||||
|
- `config`: Manage configuration.
|
||||||
|
- `print`: Print files.
|
||||||
|
|
||||||
|
### Command Options
|
||||||
|
|
||||||
|
#### `ask` Command Options
|
||||||
|
|
||||||
|
- `-q`, `--question`: Question to ask (required).
|
||||||
- `-m`, `--max-tokens`: Max tokens to use.
|
- `-m`, `--max-tokens`: Max tokens to use.
|
||||||
- `-T`, `--temperature`: Temperature to use.
|
- `-T`, `--temperature`: Temperature to use.
|
||||||
- `-M`, `--model`: Model to use.
|
- `-M`, `--model`: Model to use.
|
||||||
- `-n`, `--number`: Number of answers to produce (default is 3).
|
- `-n`, `--number`: Number of answers to produce (default is 3).
|
||||||
|
- `-s`, `--source`: Add content of a file to the query.
|
||||||
|
- `-S`, `--only-source-code`: Add pure source code to the chat history.
|
||||||
- `-t`, `--tags`: List of tag names.
|
- `-t`, `--tags`: List of tag names.
|
||||||
- `-e`, `--extags`: List of tag names to exclude.
|
- `-e`, `--extags`: List of tag names to exclude.
|
||||||
- `-o`, `--output-tags`: List of output tag names (default is the input tags).
|
- `-o`, `--output-tags`: List of output tag names (default is the input tags).
|
||||||
|
- `-a`, `--match-all-tags`: All given tags must match when selecting chat history entries.
|
||||||
|
|
||||||
|
#### `hist` Command Options
|
||||||
|
|
||||||
|
- `-d`, `--dump`: Print chat history as Python structure.
|
||||||
|
- `-w`, `--with-tags`: Print chat history with tags.
|
||||||
|
- `-W`, `--with-files`: Print chat history with filenames.
|
||||||
|
- `-S`, `--only-source-code`: Print only source code.
|
||||||
|
- `-t`, `--tags`: List of tag names.
|
||||||
|
- `-e`, `--extags`: List of tag names to exclude.
|
||||||
|
- `-a`, `--match-all-tags`: All given tags must match when selecting chat history entries.
|
||||||
|
|
||||||
|
#### `tag` Command Options
|
||||||
|
|
||||||
|
- `-l`, `--list`: List all tags and their frequency.
|
||||||
|
|
||||||
|
#### `config` Command Options
|
||||||
|
|
||||||
|
- `-l`, `--list-models`: List all available models.
|
||||||
|
- `-m`, `--print-model`: Print the currently configured model.
|
||||||
|
- `-M`, `--model`: Set model in the config file.
|
||||||
|
|
||||||
|
#### `print` Command Options
|
||||||
|
|
||||||
|
- `-f`, `--file`: File to print (required).
|
||||||
|
- `-S`, `--only-source-code`: Print only source code.
|
||||||
|
|
||||||
### Examples
|
### Examples
|
||||||
|
|
||||||
1. Print the contents of a YAML file:
|
1. Ask a question:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmm -p example.yaml
|
cmm ask -q "What is the meaning of life?" -t philosophy -e religion
|
||||||
```
|
```
|
||||||
|
|
||||||
2. Ask a question:
|
2. Display the chat history:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmm -q "What is the meaning of life?" -t philosophy -e religion
|
cmm hist
|
||||||
```
|
```
|
||||||
|
|
||||||
3. Display the chat history as a Python structure:
|
3. Filter chat history by tags:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmm -D
|
cmm hist -t tag1 tag2
|
||||||
```
|
```
|
||||||
|
|
||||||
4. Display the chat history as readable text:
|
4. Exclude chat history by tags:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmm -d
|
cmm hist -e tag3 tag4
|
||||||
```
|
```
|
||||||
|
|
||||||
5. Filter chat history by tags:
|
5. List all tags and their frequency:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmm -d -t tag1 tag2
|
cmm tag -l
|
||||||
```
|
```
|
||||||
|
|
||||||
6. Exclude chat history by tags:
|
6. Print the contents of a file:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cmm -d -e tag3 tag4
|
cmm print -f example.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|||||||
@ -5,7 +5,7 @@ def openai_api_key(api_key: str) -> None:
|
|||||||
openai.api_key = api_key
|
openai.api_key = api_key
|
||||||
|
|
||||||
|
|
||||||
def display_models() -> None:
|
def print_models() -> None:
|
||||||
not_ready = []
|
not_ready = []
|
||||||
for engine in sorted(openai.Engine.list()['data'], key=lambda x: x['id']):
|
for engine in sorted(openai.Engine.list()['data'], key=lambda x: x['id']):
|
||||||
if engine['ready']:
|
if engine['ready']:
|
||||||
|
|||||||
@ -7,38 +7,33 @@ import sys
|
|||||||
import argcomplete
|
import argcomplete
|
||||||
import argparse
|
import argparse
|
||||||
import pathlib
|
import pathlib
|
||||||
from .utils import terminal_width, process_tags, display_chat, display_source_code, display_tags_frequency
|
from .utils import terminal_width, print_tag_args, print_chat_hist, display_source_code, print_tags_frequency, ConfigType
|
||||||
from .storage import save_answers, create_chat, get_tags, get_tags_unique, read_file, dump_data
|
from .storage import save_answers, create_chat_hist, get_tags, get_tags_unique, read_file, read_config, write_config, dump_data
|
||||||
from .api_client import ai, openai_api_key, display_models
|
from .api_client import ai, openai_api_key, print_models
|
||||||
from itertools import zip_longest
|
from itertools import zip_longest
|
||||||
|
|
||||||
|
default_config = '.config.yaml'
|
||||||
def run_print_command(args: argparse.Namespace, config: dict) -> None:
|
|
||||||
fname = pathlib.Path(args.print)
|
|
||||||
if fname.suffix == '.yaml':
|
|
||||||
with open(args.print, 'r') as f:
|
|
||||||
data = yaml.load(f, Loader=yaml.FullLoader)
|
|
||||||
elif fname.suffix == '.txt':
|
|
||||||
data = read_file(fname)
|
|
||||||
else:
|
|
||||||
print(f"Unknown file type: {args.print}")
|
|
||||||
sys.exit(1)
|
|
||||||
if args.only_source_code:
|
|
||||||
display_source_code(data['answer'])
|
|
||||||
else:
|
|
||||||
print(dump_data(data).strip())
|
|
||||||
|
|
||||||
|
|
||||||
def process_and_display_chat(args: argparse.Namespace,
|
def tags_completer(prefix, parsed_args, **kwargs):
|
||||||
config: dict,
|
with open(parsed_args.config, 'r') as f:
|
||||||
dump: bool = False
|
config = yaml.load(f, Loader=yaml.FullLoader)
|
||||||
|
return get_tags_unique(config, prefix)
|
||||||
|
|
||||||
|
|
||||||
|
def create_question_with_hist(args: argparse.Namespace,
|
||||||
|
config: ConfigType,
|
||||||
) -> tuple[list[dict[str, str]], str, list[str]]:
|
) -> tuple[list[dict[str, str]], str, list[str]]:
|
||||||
|
"""
|
||||||
|
Creates the "AI request", including the question and chat history as determined
|
||||||
|
by the specified tags.
|
||||||
|
"""
|
||||||
tags = args.tags or []
|
tags = args.tags or []
|
||||||
extags = args.extags or []
|
extags = args.extags or []
|
||||||
otags = args.output_tags or []
|
otags = args.output_tags or []
|
||||||
|
|
||||||
if not args.only_source_code:
|
if not args.only_source_code:
|
||||||
process_tags(tags, extags, otags)
|
print_tag_args(tags, extags, otags)
|
||||||
|
|
||||||
question_parts = []
|
question_parts = []
|
||||||
question_list = args.question if args.question is not None else []
|
question_list = args.question if args.question is not None else []
|
||||||
@ -55,25 +50,50 @@ def process_and_display_chat(args: argparse.Namespace,
|
|||||||
question_parts.append(f"```\n{r.read().strip()}\n```")
|
question_parts.append(f"```\n{r.read().strip()}\n```")
|
||||||
|
|
||||||
full_question = '\n\n'.join(question_parts)
|
full_question = '\n\n'.join(question_parts)
|
||||||
chat = create_chat(full_question, tags, extags, config,
|
chat = create_chat_hist(full_question, tags, extags, config,
|
||||||
args.match_all_tags, args.with_tags,
|
args.match_all_tags, False, False)
|
||||||
args.with_file)
|
|
||||||
display_chat(chat, dump, args.only_source_code)
|
|
||||||
return chat, full_question, tags
|
return chat, full_question, tags
|
||||||
|
|
||||||
|
|
||||||
def process_and_display_tags(args: argparse.Namespace,
|
def tag_cmd(args: argparse.Namespace, config: ConfigType) -> None:
|
||||||
config: dict,
|
"""
|
||||||
dump: bool = False
|
Handler for the 'tag' command.
|
||||||
) -> None:
|
"""
|
||||||
display_tags_frequency(get_tags(config, None), dump)
|
if args.list:
|
||||||
|
print_tags_frequency(get_tags(config, None))
|
||||||
|
|
||||||
|
|
||||||
def handle_question(args: argparse.Namespace,
|
def config_cmd(args: argparse.Namespace, config: ConfigType) -> None:
|
||||||
config: dict,
|
"""
|
||||||
dump: bool = False
|
Handler for the 'config' command.
|
||||||
) -> None:
|
"""
|
||||||
chat, question, tags = process_and_display_chat(args, config, dump)
|
if type(config['openai']) is not dict:
|
||||||
|
raise RuntimeError('Configuration openai is not a dict.')
|
||||||
|
|
||||||
|
if args.list_models:
|
||||||
|
print_models()
|
||||||
|
elif args.print_model:
|
||||||
|
print(config['openai']['model'])
|
||||||
|
elif args.model:
|
||||||
|
config['openai']['model'] = args.model
|
||||||
|
write_config(args.config, config)
|
||||||
|
|
||||||
|
|
||||||
|
def ask_cmd(args: argparse.Namespace, config: ConfigType) -> None:
|
||||||
|
"""
|
||||||
|
Handler for the 'ask' command.
|
||||||
|
"""
|
||||||
|
if type(config['openai']) is not dict:
|
||||||
|
raise RuntimeError('Configuration openai is not a dict.')
|
||||||
|
config_openai = config['openai']
|
||||||
|
if args.max_tokens:
|
||||||
|
config_openai['max_tokens'] = args.max_tokens
|
||||||
|
if args.temperature:
|
||||||
|
config_openai['temperature'] = args.temperature
|
||||||
|
if args.model:
|
||||||
|
config_openai['model'] = args.model
|
||||||
|
chat, question, tags = create_question_with_hist(args, config)
|
||||||
|
print_chat_hist(chat, False, args.only_source_code)
|
||||||
otags = args.output_tags or []
|
otags = args.output_tags or []
|
||||||
answers, usage = ai(chat, config, args.number)
|
answers, usage = ai(chat, config, args.number)
|
||||||
save_answers(question, answers, tags, otags, config)
|
save_answers(question, answers, tags, otags, config)
|
||||||
@ -81,43 +101,120 @@ def handle_question(args: argparse.Namespace,
|
|||||||
print(f"Usage: {usage}")
|
print(f"Usage: {usage}")
|
||||||
|
|
||||||
|
|
||||||
def tags_completer(prefix, parsed_args, **kwargs):
|
def hist_cmd(args: argparse.Namespace, config: ConfigType) -> None:
|
||||||
with open(parsed_args.config, 'r') as f:
|
"""
|
||||||
config = yaml.load(f, Loader=yaml.FullLoader)
|
Handler for the 'hist' command.
|
||||||
return get_tags_unique(config, prefix)
|
"""
|
||||||
|
tags = args.tags or []
|
||||||
|
extags = args.extags or []
|
||||||
|
|
||||||
|
chat = create_chat_hist(None, tags, extags, config,
|
||||||
|
args.match_all_tags,
|
||||||
|
args.with_tags,
|
||||||
|
args.with_files)
|
||||||
|
print_chat_hist(chat, args.dump, args.only_source_code)
|
||||||
|
|
||||||
|
|
||||||
|
def print_cmd(args: argparse.Namespace, config: ConfigType) -> None:
|
||||||
|
"""
|
||||||
|
Handler for the 'print' command.
|
||||||
|
"""
|
||||||
|
fname = pathlib.Path(args.file)
|
||||||
|
if fname.suffix == '.yaml':
|
||||||
|
with open(args.file, 'r') as f:
|
||||||
|
data = yaml.load(f, Loader=yaml.FullLoader)
|
||||||
|
elif fname.suffix == '.txt':
|
||||||
|
data = read_file(fname)
|
||||||
|
else:
|
||||||
|
print(f"Unknown file type: {args.file}")
|
||||||
|
sys.exit(1)
|
||||||
|
if args.only_source_code:
|
||||||
|
display_source_code(data['answer'])
|
||||||
|
else:
|
||||||
|
print(dump_data(data).strip())
|
||||||
|
|
||||||
|
|
||||||
def create_parser() -> argparse.ArgumentParser:
|
def create_parser() -> argparse.ArgumentParser:
|
||||||
default_config = '.config.yaml'
|
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="ChatMastermind is a Python application that automates conversation with AI")
|
description="ChatMastermind is a Python application that automates conversation with AI")
|
||||||
group = parser.add_mutually_exclusive_group(required=True)
|
|
||||||
group.add_argument('-p', '--print', help='File to print')
|
|
||||||
group.add_argument('-q', '--question', nargs='*', help='Question to ask')
|
|
||||||
group.add_argument('-D', '--chat-dump', help="Print chat history as Python structure", action='store_true')
|
|
||||||
group.add_argument('-d', '--chat', help="Print chat history as readable text", action='store_true')
|
|
||||||
group.add_argument('-l', '--list-tags', help="List all tags and their frequency", action='store_true')
|
|
||||||
group.add_argument('-L', '--list-models', help="List all available models", action='store_true')
|
|
||||||
parser.add_argument('-c', '--config', help='Config file name.', default=default_config)
|
parser.add_argument('-c', '--config', help='Config file name.', default=default_config)
|
||||||
parser.add_argument('-m', '--max-tokens', help='Max tokens to use', type=int)
|
|
||||||
parser.add_argument('-T', '--temperature', help='Temperature to use', type=float)
|
# subcommand-parser
|
||||||
parser.add_argument('-M', '--model', help='Model to use')
|
cmdparser = parser.add_subparsers(dest='command',
|
||||||
parser.add_argument('-n', '--number', help='Number of answers to produce', type=int, default=1)
|
title='commands',
|
||||||
parser.add_argument('-s', '--source', nargs='*', help='Source add content of a file to the query')
|
description='supported commands',
|
||||||
parser.add_argument('-S', '--only-source-code', help='Print only source code', action='store_true')
|
required=True)
|
||||||
parser.add_argument('-w', '--with-tags', help="Print chat history with tags.", action='store_true')
|
|
||||||
parser.add_argument('-W', '--with-file',
|
# a parent parser for all commands that support tag selection
|
||||||
help="Print chat history with filename.",
|
tag_parser = argparse.ArgumentParser(add_help=False)
|
||||||
|
tag_arg = tag_parser.add_argument('-t', '--tags', nargs='+',
|
||||||
|
help='List of tag names', metavar='TAGS')
|
||||||
|
tag_arg.completer = tags_completer # type: ignore
|
||||||
|
extag_arg = tag_parser.add_argument('-e', '--extags', nargs='+',
|
||||||
|
help='List of tag names to exclude', metavar='EXTAGS')
|
||||||
|
extag_arg.completer = tags_completer # type: ignore
|
||||||
|
otag_arg = tag_parser.add_argument('-o', '--output-tags', nargs='+',
|
||||||
|
help='List of output tag names, default is input', metavar='OTAGS')
|
||||||
|
otag_arg.completer = tags_completer # type: ignore
|
||||||
|
tag_parser.add_argument('-a', '--match-all-tags',
|
||||||
|
help="All given tags must match when selecting chat history entries",
|
||||||
action='store_true')
|
action='store_true')
|
||||||
parser.add_argument('-a', '--match-all-tags',
|
# enable autocompletion for tags
|
||||||
help="All given tags must match when selecting chat history entries.",
|
|
||||||
|
# 'ask' command parser
|
||||||
|
ask_cmd_parser = cmdparser.add_parser('ask', parents=[tag_parser],
|
||||||
|
help="Ask a question.")
|
||||||
|
ask_cmd_parser.set_defaults(func=ask_cmd)
|
||||||
|
ask_cmd_parser.add_argument('-q', '--question', nargs='+', help='Question to ask',
|
||||||
|
required=True)
|
||||||
|
ask_cmd_parser.add_argument('-m', '--max-tokens', help='Max tokens to use', type=int)
|
||||||
|
ask_cmd_parser.add_argument('-T', '--temperature', help='Temperature to use', type=float)
|
||||||
|
ask_cmd_parser.add_argument('-M', '--model', help='Model to use')
|
||||||
|
ask_cmd_parser.add_argument('-n', '--number', help='Number of answers to produce', type=int,
|
||||||
|
default=1)
|
||||||
|
ask_cmd_parser.add_argument('-s', '--source', nargs='+', help='Source add content of a file to the query')
|
||||||
|
ask_cmd_parser.add_argument('-S', '--only-source-code', help='Add pure source code to the chat history',
|
||||||
action='store_true')
|
action='store_true')
|
||||||
tags_arg = parser.add_argument('-t', '--tags', nargs='*', help='List of tag names', metavar='TAGS')
|
|
||||||
tags_arg.completer = tags_completer # type: ignore
|
# 'hist' command parser
|
||||||
extags_arg = parser.add_argument('-e', '--extags', nargs='*', help='List of tag names to exclude', metavar='EXTAGS')
|
hist_cmd_parser = cmdparser.add_parser('hist', parents=[tag_parser],
|
||||||
extags_arg.completer = tags_completer # type: ignore
|
help="Print chat history.")
|
||||||
otags_arg = parser.add_argument('-o', '--output-tags', nargs='*', help='List of output tag names, default is input', metavar='OTAGS')
|
hist_cmd_parser.set_defaults(func=hist_cmd)
|
||||||
otags_arg.completer = tags_completer # type: ignore
|
hist_cmd_parser.add_argument('-d', '--dump', help="Print chat history as Python structure",
|
||||||
|
action='store_true')
|
||||||
|
hist_cmd_parser.add_argument('-w', '--with-tags', help="Print chat history with tags.",
|
||||||
|
action='store_true')
|
||||||
|
hist_cmd_parser.add_argument('-W', '--with-files', help="Print chat history with filenames.",
|
||||||
|
action='store_true')
|
||||||
|
hist_cmd_parser.add_argument('-S', '--only-source-code', help='Print only source code',
|
||||||
|
action='store_true')
|
||||||
|
|
||||||
|
# 'tag' command parser
|
||||||
|
tag_cmd_parser = cmdparser.add_parser('tag',
|
||||||
|
help="Manage tags.")
|
||||||
|
tag_cmd_parser.set_defaults(func=tag_cmd)
|
||||||
|
tag_cmd_parser.add_argument('-l', '--list', help="List all tags and their frequency",
|
||||||
|
action='store_true')
|
||||||
|
|
||||||
|
# 'config' command parser
|
||||||
|
config_cmd_parser = cmdparser.add_parser('config',
|
||||||
|
help="Manage configuration")
|
||||||
|
config_cmd_parser.set_defaults(func=config_cmd)
|
||||||
|
config_group = config_cmd_parser.add_mutually_exclusive_group(required=True)
|
||||||
|
config_group.add_argument('-l', '--list-models', help="List all available models",
|
||||||
|
action='store_true')
|
||||||
|
config_group.add_argument('-m', '--print-model', help="Print the currently configured model",
|
||||||
|
action='store_true')
|
||||||
|
config_group.add_argument('-M', '--model', help="Set model in the config file")
|
||||||
|
|
||||||
|
# 'print' command parser
|
||||||
|
print_cmd_parser = cmdparser.add_parser('print',
|
||||||
|
help="Print files.")
|
||||||
|
print_cmd_parser.set_defaults(func=print_cmd)
|
||||||
|
print_cmd_parser.add_argument('-f', '--file', help='File to print', required=True)
|
||||||
|
print_cmd_parser.add_argument('-S', '--only-source-code', help='Print only source code',
|
||||||
|
action='store_true')
|
||||||
|
|
||||||
argcomplete.autocomplete(parser)
|
argcomplete.autocomplete(parser)
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
@ -125,33 +222,15 @@ def create_parser() -> argparse.ArgumentParser:
|
|||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = create_parser()
|
parser = create_parser()
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
command = parser.parse_args()
|
||||||
|
config = read_config(args.config)
|
||||||
|
|
||||||
with open(args.config, 'r') as f:
|
if type(config['openai']) is dict and type(config['openai']['api_key']) is str:
|
||||||
config = yaml.load(f, Loader=yaml.FullLoader)
|
|
||||||
|
|
||||||
openai_api_key(config['openai']['api_key'])
|
openai_api_key(config['openai']['api_key'])
|
||||||
|
else:
|
||||||
|
raise RuntimeError("Configuration openai.api_key is wrong.")
|
||||||
|
|
||||||
if args.max_tokens:
|
command.func(command, config)
|
||||||
config['openai']['max_tokens'] = args.max_tokens
|
|
||||||
|
|
||||||
if args.temperature:
|
|
||||||
config['openai']['temperature'] = args.temperature
|
|
||||||
|
|
||||||
if args.model:
|
|
||||||
config['openai']['model'] = args.model
|
|
||||||
|
|
||||||
if args.print:
|
|
||||||
run_print_command(args, config)
|
|
||||||
elif args.question:
|
|
||||||
handle_question(args, config)
|
|
||||||
elif args.chat_dump:
|
|
||||||
process_and_display_chat(args, config, dump=True)
|
|
||||||
elif args.chat:
|
|
||||||
process_and_display_chat(args, config)
|
|
||||||
elif args.list_tags:
|
|
||||||
process_and_display_tags(args, config)
|
|
||||||
elif args.list_models:
|
|
||||||
display_models()
|
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
import yaml
|
import yaml
|
||||||
import io
|
import io
|
||||||
import pathlib
|
import pathlib
|
||||||
from .utils import terminal_width, append_message, message_to_chat
|
from .utils import terminal_width, append_message, message_to_chat, ConfigType
|
||||||
from typing import List, Dict, Any, Optional
|
from typing import List, Dict, Any, Optional
|
||||||
|
|
||||||
|
|
||||||
@ -22,6 +22,17 @@ def read_file(fname: pathlib.Path, tags_only: bool = False) -> Dict[str, Any]:
|
|||||||
"file": fname.name}
|
"file": fname.name}
|
||||||
|
|
||||||
|
|
||||||
|
def read_config(path: str) -> ConfigType:
|
||||||
|
with open(path, 'r') as f:
|
||||||
|
config = yaml.load(f, Loader=yaml.FullLoader)
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def write_config(path: str, config: ConfigType) -> None:
|
||||||
|
with open(path, 'w') as f:
|
||||||
|
yaml.dump(config, f)
|
||||||
|
|
||||||
|
|
||||||
def dump_data(data: Dict[str, Any]) -> str:
|
def dump_data(data: Dict[str, Any]) -> str:
|
||||||
with io.StringIO() as fd:
|
with io.StringIO() as fd:
|
||||||
fd.write(f'TAGS: {" ".join(data["tags"])}\n')
|
fd.write(f'TAGS: {" ".join(data["tags"])}\n')
|
||||||
@ -41,11 +52,11 @@ def save_answers(question: str,
|
|||||||
answers: list[str],
|
answers: list[str],
|
||||||
tags: list[str],
|
tags: list[str],
|
||||||
otags: Optional[list[str]],
|
otags: Optional[list[str]],
|
||||||
config: Dict[str, Any]
|
config: ConfigType
|
||||||
) -> None:
|
) -> None:
|
||||||
wtags = otags or tags
|
wtags = otags or tags
|
||||||
num, inum = 0, 0
|
num, inum = 0, 0
|
||||||
next_fname = pathlib.Path(config['db']) / '.next'
|
next_fname = pathlib.Path(str(config['db'])) / '.next'
|
||||||
try:
|
try:
|
||||||
with open(next_fname, 'r') as f:
|
with open(next_fname, 'r') as f:
|
||||||
num = int(f.read())
|
num = int(f.read())
|
||||||
@ -63,17 +74,17 @@ def save_answers(question: str,
|
|||||||
f.write(f'{num}')
|
f.write(f'{num}')
|
||||||
|
|
||||||
|
|
||||||
def create_chat(question: Optional[str],
|
def create_chat_hist(question: Optional[str],
|
||||||
tags: Optional[List[str]],
|
tags: Optional[List[str]],
|
||||||
extags: Optional[List[str]],
|
extags: Optional[List[str]],
|
||||||
config: Dict[str, Any],
|
config: ConfigType,
|
||||||
match_all_tags: bool = False,
|
match_all_tags: bool = False,
|
||||||
with_tags: bool = False,
|
with_tags: bool = False,
|
||||||
with_file: bool = False
|
with_file: bool = False
|
||||||
) -> List[Dict[str, str]]:
|
) -> List[Dict[str, str]]:
|
||||||
chat: List[Dict[str, str]] = []
|
chat: List[Dict[str, str]] = []
|
||||||
append_message(chat, 'system', config['system'].strip())
|
append_message(chat, 'system', str(config['system']).strip())
|
||||||
for file in sorted(pathlib.Path(config['db']).iterdir()):
|
for file in sorted(pathlib.Path(str(config['db'])).iterdir()):
|
||||||
if file.suffix == '.yaml':
|
if file.suffix == '.yaml':
|
||||||
with open(file, 'r') as f:
|
with open(file, 'r') as f:
|
||||||
data = yaml.load(f, Loader=yaml.FullLoader)
|
data = yaml.load(f, Loader=yaml.FullLoader)
|
||||||
@ -97,9 +108,9 @@ def create_chat(question: Optional[str],
|
|||||||
return chat
|
return chat
|
||||||
|
|
||||||
|
|
||||||
def get_tags(config: Dict[str, Any], prefix: Optional[str]) -> List[str]:
|
def get_tags(config: ConfigType, prefix: Optional[str]) -> List[str]:
|
||||||
result = []
|
result = []
|
||||||
for file in sorted(pathlib.Path(config['db']).iterdir()):
|
for file in sorted(pathlib.Path(str(config['db'])).iterdir()):
|
||||||
if file.suffix == '.yaml':
|
if file.suffix == '.yaml':
|
||||||
with open(file, 'r') as f:
|
with open(file, 'r') as f:
|
||||||
data = yaml.load(f, Loader=yaml.FullLoader)
|
data = yaml.load(f, Loader=yaml.FullLoader)
|
||||||
@ -116,5 +127,5 @@ def get_tags(config: Dict[str, Any], prefix: Optional[str]) -> List[str]:
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def get_tags_unique(config: Dict[str, Any], prefix: Optional[str]) -> List[str]:
|
def get_tags_unique(config: ConfigType, prefix: Optional[str]) -> List[str]:
|
||||||
return list(set(get_tags(config, prefix)))
|
return list(set(get_tags(config, prefix)))
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import shutil
|
import shutil
|
||||||
from pprint import PrettyPrinter
|
from pprint import PrettyPrinter
|
||||||
from typing import List, Dict
|
|
||||||
|
ConfigType = dict[str, str | dict[str, str | int | float]]
|
||||||
|
|
||||||
|
|
||||||
def terminal_width() -> int:
|
def terminal_width() -> int:
|
||||||
@ -11,7 +12,10 @@ def pp(*args, **kwargs) -> None:
|
|||||||
return PrettyPrinter(width=terminal_width()).pprint(*args, **kwargs)
|
return PrettyPrinter(width=terminal_width()).pprint(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def process_tags(tags: list[str], extags: list[str], otags: list[str]) -> None:
|
def print_tag_args(tags: list[str], extags: list[str], otags: list[str]) -> None:
|
||||||
|
"""
|
||||||
|
Prints the tags specified in the given args.
|
||||||
|
"""
|
||||||
printed_messages = []
|
printed_messages = []
|
||||||
|
|
||||||
if tags:
|
if tags:
|
||||||
@ -26,15 +30,15 @@ def process_tags(tags: list[str], extags: list[str], otags: list[str]) -> None:
|
|||||||
print()
|
print()
|
||||||
|
|
||||||
|
|
||||||
def append_message(chat: List[Dict[str, str]],
|
def append_message(chat: list[dict[str, str]],
|
||||||
role: str,
|
role: str,
|
||||||
content: str
|
content: str
|
||||||
) -> None:
|
) -> None:
|
||||||
chat.append({'role': role, 'content': content.replace("''", "'")})
|
chat.append({'role': role, 'content': content.replace("''", "'")})
|
||||||
|
|
||||||
|
|
||||||
def message_to_chat(message: Dict[str, str],
|
def message_to_chat(message: dict[str, str],
|
||||||
chat: List[Dict[str, str]],
|
chat: list[dict[str, str]],
|
||||||
with_tags: bool = False,
|
with_tags: bool = False,
|
||||||
with_file: bool = False
|
with_file: bool = False
|
||||||
) -> None:
|
) -> None:
|
||||||
@ -57,7 +61,7 @@ def display_source_code(content: str) -> None:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def display_chat(chat, dump=False, source_code=False) -> None:
|
def print_chat_hist(chat, dump=False, source_code=False) -> None:
|
||||||
if dump:
|
if dump:
|
||||||
pp(chat)
|
pp(chat)
|
||||||
return
|
return
|
||||||
@ -75,9 +79,6 @@ def display_chat(chat, dump=False, source_code=False) -> None:
|
|||||||
print(f"{message['role'].upper()}: {message['content']}")
|
print(f"{message['role'].upper()}: {message['content']}")
|
||||||
|
|
||||||
|
|
||||||
def display_tags_frequency(tags: List[str], dump=False) -> None:
|
def print_tags_frequency(tags: list[str]) -> None:
|
||||||
if dump:
|
|
||||||
pp(tags)
|
|
||||||
return
|
|
||||||
for tag in sorted(set(tags)):
|
for tag in sorted(set(tags)):
|
||||||
print(f"- {tag}: {tags.count(tag)}")
|
print(f"- {tag}: {tags.count(tag)}")
|
||||||
|
|||||||
@ -3,11 +3,11 @@ import io
|
|||||||
import pathlib
|
import pathlib
|
||||||
import argparse
|
import argparse
|
||||||
from chatmastermind.utils import terminal_width
|
from chatmastermind.utils import terminal_width
|
||||||
from chatmastermind.main import create_parser, handle_question
|
from chatmastermind.main import create_parser, ask_cmd
|
||||||
from chatmastermind.api_client import ai
|
from chatmastermind.api_client import ai
|
||||||
from chatmastermind.storage import create_chat, save_answers, dump_data
|
from chatmastermind.storage import create_chat_hist, save_answers, dump_data
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
from unittest.mock import patch, MagicMock, Mock
|
from unittest.mock import patch, MagicMock, Mock, ANY
|
||||||
|
|
||||||
|
|
||||||
class TestCreateChat(unittest.TestCase):
|
class TestCreateChat(unittest.TestCase):
|
||||||
@ -30,7 +30,7 @@ class TestCreateChat(unittest.TestCase):
|
|||||||
{'question': 'test_content', 'answer': 'some answer',
|
{'question': 'test_content', 'answer': 'some answer',
|
||||||
'tags': ['test_tag']}))
|
'tags': ['test_tag']}))
|
||||||
|
|
||||||
test_chat = create_chat(self.question, self.tags, None, self.config)
|
test_chat = create_chat_hist(self.question, self.tags, None, self.config)
|
||||||
|
|
||||||
self.assertEqual(len(test_chat), 4)
|
self.assertEqual(len(test_chat), 4)
|
||||||
self.assertEqual(test_chat[0],
|
self.assertEqual(test_chat[0],
|
||||||
@ -52,7 +52,7 @@ class TestCreateChat(unittest.TestCase):
|
|||||||
{'question': 'test_content', 'answer': 'some answer',
|
{'question': 'test_content', 'answer': 'some answer',
|
||||||
'tags': ['other_tag']}))
|
'tags': ['other_tag']}))
|
||||||
|
|
||||||
test_chat = create_chat(self.question, self.tags, None, self.config)
|
test_chat = create_chat_hist(self.question, self.tags, None, self.config)
|
||||||
|
|
||||||
self.assertEqual(len(test_chat), 2)
|
self.assertEqual(len(test_chat), 2)
|
||||||
self.assertEqual(test_chat[0],
|
self.assertEqual(test_chat[0],
|
||||||
@ -75,7 +75,7 @@ class TestCreateChat(unittest.TestCase):
|
|||||||
'tags': ['test_tag2']})),
|
'tags': ['test_tag2']})),
|
||||||
)
|
)
|
||||||
|
|
||||||
test_chat = create_chat(self.question, [], None, self.config)
|
test_chat = create_chat_hist(self.question, [], None, self.config)
|
||||||
|
|
||||||
self.assertEqual(len(test_chat), 6)
|
self.assertEqual(len(test_chat), 6)
|
||||||
self.assertEqual(test_chat[0],
|
self.assertEqual(test_chat[0],
|
||||||
@ -102,6 +102,9 @@ class TestHandleQuestion(unittest.TestCase):
|
|||||||
source=None,
|
source=None,
|
||||||
only_source_code=False,
|
only_source_code=False,
|
||||||
number=3,
|
number=3,
|
||||||
|
max_tokens=None,
|
||||||
|
temperature=None,
|
||||||
|
model=None,
|
||||||
match_all_tags=False,
|
match_all_tags=False,
|
||||||
with_tags=False,
|
with_tags=False,
|
||||||
with_file=False,
|
with_file=False,
|
||||||
@ -109,28 +112,33 @@ class TestHandleQuestion(unittest.TestCase):
|
|||||||
self.config = {
|
self.config = {
|
||||||
'db': 'test_files',
|
'db': 'test_files',
|
||||||
'setting1': 'value1',
|
'setting1': 'value1',
|
||||||
'setting2': 'value2'
|
'setting2': 'value2',
|
||||||
|
'openai': {},
|
||||||
}
|
}
|
||||||
|
|
||||||
@patch("chatmastermind.main.create_chat", return_value="test_chat")
|
@patch("chatmastermind.main.create_chat_hist", return_value="test_chat")
|
||||||
@patch("chatmastermind.main.process_tags")
|
@patch("chatmastermind.main.print_tag_args")
|
||||||
|
@patch("chatmastermind.main.print_chat_hist")
|
||||||
@patch("chatmastermind.main.ai", return_value=(["answer1", "answer2", "answer3"], "test_usage"))
|
@patch("chatmastermind.main.ai", return_value=(["answer1", "answer2", "answer3"], "test_usage"))
|
||||||
@patch("chatmastermind.utils.pp")
|
@patch("chatmastermind.utils.pp")
|
||||||
@patch("builtins.print")
|
@patch("builtins.print")
|
||||||
def test_handle_question(self, mock_print, mock_pp, mock_ai,
|
def test_ask_cmd(self, mock_print, mock_pp, mock_ai,
|
||||||
mock_process_tags, mock_create_chat):
|
mock_print_chat_hist, mock_print_tag_args,
|
||||||
|
mock_create_chat_hist):
|
||||||
open_mock = MagicMock()
|
open_mock = MagicMock()
|
||||||
with patch("chatmastermind.storage.open", open_mock):
|
with patch("chatmastermind.storage.open", open_mock):
|
||||||
handle_question(self.args, self.config, True)
|
ask_cmd(self.args, self.config)
|
||||||
mock_process_tags.assert_called_once_with(self.args.tags,
|
mock_print_tag_args.assert_called_once_with(self.args.tags,
|
||||||
self.args.extags,
|
self.args.extags,
|
||||||
[])
|
[])
|
||||||
mock_create_chat.assert_called_once_with(self.question,
|
mock_create_chat_hist.assert_called_once_with(self.question,
|
||||||
self.args.tags,
|
self.args.tags,
|
||||||
self.args.extags,
|
self.args.extags,
|
||||||
self.config,
|
self.config,
|
||||||
False, False, False)
|
False, False, False)
|
||||||
mock_pp.assert_called_once_with("test_chat")
|
mock_print_chat_hist.assert_called_once_with('test_chat',
|
||||||
|
False,
|
||||||
|
self.args.only_source_code)
|
||||||
mock_ai.assert_called_with("test_chat",
|
mock_ai.assert_called_with("test_chat",
|
||||||
self.config,
|
self.config,
|
||||||
self.args.number)
|
self.args.number)
|
||||||
@ -205,15 +213,15 @@ class TestAI(unittest.TestCase):
|
|||||||
|
|
||||||
class TestCreateParser(unittest.TestCase):
|
class TestCreateParser(unittest.TestCase):
|
||||||
def test_create_parser(self):
|
def test_create_parser(self):
|
||||||
with patch('argparse.ArgumentParser.add_mutually_exclusive_group') as mock_add_mutually_exclusive_group:
|
with patch('argparse.ArgumentParser.add_subparsers') as mock_add_subparsers:
|
||||||
mock_group = Mock()
|
mock_cmdparser = Mock()
|
||||||
mock_add_mutually_exclusive_group.return_value = mock_group
|
mock_add_subparsers.return_value = mock_cmdparser
|
||||||
parser = create_parser()
|
parser = create_parser()
|
||||||
self.assertIsInstance(parser, argparse.ArgumentParser)
|
self.assertIsInstance(parser, argparse.ArgumentParser)
|
||||||
mock_add_mutually_exclusive_group.assert_called_once_with(required=True)
|
mock_add_subparsers.assert_called_once_with(dest='command', title='commands', description='supported commands', required=True)
|
||||||
mock_group.add_argument.assert_any_call('-p', '--print', help='File to print')
|
mock_cmdparser.add_parser.assert_any_call('ask', parents=ANY, help=ANY)
|
||||||
mock_group.add_argument.assert_any_call('-q', '--question', nargs='*', help='Question to ask')
|
mock_cmdparser.add_parser.assert_any_call('hist', parents=ANY, help=ANY)
|
||||||
mock_group.add_argument.assert_any_call('-D', '--chat-dump', help="Print chat history as Python structure", action='store_true')
|
mock_cmdparser.add_parser.assert_any_call('tag', help=ANY)
|
||||||
mock_group.add_argument.assert_any_call('-d', '--chat', help="Print chat history as readable text", action='store_true')
|
mock_cmdparser.add_parser.assert_any_call('config', help=ANY)
|
||||||
|
mock_cmdparser.add_parser.assert_any_call('print', help=ANY)
|
||||||
self.assertTrue('.config.yaml' in parser.get_default('config'))
|
self.assertTrue('.config.yaml' in parser.get_default('config'))
|
||||||
self.assertEqual(parser.get_default('number'), 1)
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user