BotEval Docs

conda create env -n boteval python=3.9
conda activate boteval
pip install boteval
boteval -h

#create a python-3.9 environment (if necessary) and activate
git clone git@github.com:isi-nlp/boteval.git
cd boteval
pip install -e .
boteval -h

python -m boteval -h
usage: boteval [-h] [-c FILE] [-b /prefix] [-d] [-a ADDR] [-p PORT] [-v] DIR

Deploy chat bot evaluation

positional arguments:
  DIR                   Path to task dir. See "example-chat-task"

optional arguments:
  -h, --help            show this help message and exit
  -c FILE, --config FILE
                        Path to config file. Default is <task-dir>/conf.yml (default: None)
  -b /prefix, --base /prefix
                        Base path prefix for all url routes. Eg: /boteval (default: None)
  -d, --debug           Run Flask server in debug mode (default: False)
  -a ADDR, --addr ADDR  Address to bind to (default: 0.0.0.0)
  -p PORT, --port PORT  port to run server on (default: 7070)
  -v, --version         show version number and exit

# create an example task directory
boteval-quickstart example-chat-task/

# run demo task
boteval -d example-chat-task/

example-chat-task/
├── conf.yml             (1)
├── chat_topics.json
├── __init__.py          (2)
├── instructions.html
└── user-agreement.html

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
chatbot:
  display_name: 'Moderator'
  topics_file: chat_topics.json
  bot_name: hf-transformers   (1)
  bot_args:                   (1)
    model_name: facebook/blenderbot_small-90M

onboarding: (2)
  agreement_file: user-agreement.html
  instructions_file: instructions.html
  checkboxes:
    instructions_read: I have read the instructions.
    iam_adult: I am 18 years or older and I understand that I may have to read and write using toxic language.

ratings: (3)
  - question: 'How Coherent was the conversation?'
    choices: &choices
      - Not at all
      - Mostly not
      - So-so
      - Somewhat
      - Very
  - question: 'How likely are you going to continue the conversation with the bot?'
    choices: *choices
  - question: 'To what degree did the bot convince you to change your behavior?'
    choices: *choices

limits:  (4)
  max_threads_per_user: 10
  max_threads_per_topic: &max_assignments 3
  max_turns_per_thread: 4
  reward: &reward '0.01' # dollars

flask_config: (5)
  # sqlalchemy settings https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/
  DATABASE_FILE_NAME: 'sqlite-dev-01.db'   # this will be placed in task dir
  SQLALCHEMY_TRACK_MODIFICATIONS: false

mturk: (6)
  client:
    profile: default # the [default] profile in ~/.aws/credentials file
    sandbox: true  # sandbox: false to go live
  seamless_login: true
  hit_settings:
    # https://boto3.amazonaws.com/v1/documentation/api/1.11.9/reference/services/mturk.html#MTurk.Client.create_hit
    MaxAssignments: *max_assignments
    AutoApprovalDelayInSeconds: 604800      # 7 days  = 604k sec
    LifetimeInSeconds: 1209600              # 14 days = 1.2M sec
    AssignmentDurationInSeconds: 3600       # 1 hour  = 3.6k sec
    Reward: *reward
    Title: 'Evaluate a chatbot'
    Keywords: 'chatbot,chat,research'
    Description: 'Evaluate a chat bot by talking to it for a while and receive a reward'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
from typing import Any

from boteval import log, C, registry as R
from boteval.bots import BotAgent
from boteval.transforms import BaseTransform, SpacySplitter
from boteval.model import ChatMessage


@R.register(R.BOT, name="my-dummy-bot")
class MyDummpyBot(BotAgent):

    def __init__(self, **kwargs):
        super().__init__(name="dummybot", **kwargs)
        self.args = kwargs
        log.info(f"{self.name} initialized; args={self.args}")

    def talk(self) -> dict[str, Any]:
        if not self.last_msg:
            reply = f"Hello there! I am {C.BOT_DISPLAY_NAME}."
        elif 'ping' in self.last_msg['text'].lower():
            reply = 'pong'
        else:
            reply = f"Dummy reply for -- {self.last_msg['text']}"
        return dict(text=reply)

    def hear(self, msg: dict[str, Any]):
        self.last_msg = msg


@R.register(R.TRANSFORM, name='my-transform')
class MyDummyTransform(BaseTransform):

    def __init__(self, **kwargs) -> None:
        super().__init__()
        self.args = kwargs
        self.splitter = SpacySplitter.get_instance()

    def transform(self, msg: ChatMessage) -> ChatMessage:
        try:
            text_orig = msg.text
            text = '\n'.join(self.splitter(text_orig))  + ' (transformed)' # sample transform
            msg.text = text
            msg.data['text_orig'] = text_orig
        except Exception as e:
            log.error(f'{e}')
        return msg

@R.register(R.BOT, name="my-dummy-bot")

@R.register(R.TRANSFORM, name='my-transform')

chatbot:
  display_name: 'Moderator'
  topics_file: chat_topics.json
  bot_name: my-dummy-bot
  bot_args:
    key1: val1
    key2: val2
    some_flag: true

  transforms:
    human:
      - name: my-transform
        args:
          arg1: val1
          arg2: [val2, val3]
    bot:
      - name: my-transform
        args:
          arg1: val1
          arg2: [val2, val3]

# Default server configuration
server {
        listen 80 default_server;
        listen [::]:80 default_server;

        root /var/www/html;

        # Add index.php to the list if you are using PHP
        index index.html index.htm index.nginx-debian.html;

        server_name _;

        location / {
                # First attempt to serve request as file, then
                # as directory, then fall back to displaying a 404.
                try_files $uri $uri/ =404;
        }

        location /boteval {
                proxy_pass http://127.0.0.1:7070/boteval ;
                proxy_set_header Host $http_host;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-Proto $scheme;
        }

}


server {
        root /var/www/html;
        # Add index.php to the list if you are using PHP
        index index.html index.htm index.nginx-debian.html;
        server_name dev.gowda.ai; # managed by Certbot

        location / {
                # First attempt to serve request as file, then
                # as directory, then fall back to displaying a 404.
                try_files $uri $uri/ =404;
        }

        location /boteval {
                proxy_pass http://127.0.0.1:7070/boteval;
                proxy_set_header Host $http_host;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-Proto $scheme;
                proxy_redirect http://$http_host/ https://$http_host/;
        }


    listen [::]:443 ssl ipv6only=on; # managed by Certbot
    listen 443 ssl; # managed by Certbot
    ssl_certificate /etc/letsencrypt/live/dev.gowda.ai/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/dev.gowda.ai/privkey.pem; # managed by Certbot
    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}

server {
    if ($host = dev.gowda.ai) {
        return 301 https://$host$request_uri;
    } # managed by Certbot

        listen 80 ;
        listen [::]:80 ;
    server_name dev.gowda.ai;
    return 404; # managed by Certbot

}