Home LLM的test oracle生成

# online-judge-tools/oj

# 简介

  • oj 是一个帮助解决各种在线评测问题命令。此命令会自动下载示例案例、生成其他测试案例、测试您的代码并提交。
  • 使用 python setup.py install 安装该第三方库
  • 使用介绍：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
positional arguments:
  {download,d,dl,login,l,submit,s,test,t,generate-output,g/o,generate-input,g/i,test-reactive,t/r,test-interactive,t/i}
                        for details, see "\\?\F:\Anaconda\envs\py312\Scripts\oj COMMAND --help"
    download (d, dl)    download sample cases
    login (l)           login to a service
    submit (s)          submit your solution
    test (t)            test your code
    generate-output (g/o)
                        generate output files from input and reference implementation
    generate-input (g/i)
                        generate input files from given generator
    test-reactive (t/r, test-interactive, t/i)
                        test for interactive problem

options:
  -h, --help            show this help message and exit
  -v, --verbose
  -c COOKIE, --cookie COOKIE
                        path to cookie. (default: C:\Users\JackDu\AppData\Local\online-judge-tools\online-judge-tools\cookie.jar)
  --version             print the online-judge-tools version number

tips:
  The official tutorial exists on the web: https://github.com/online-judge-tools/oj/blob/master/docs/getting-started.md

# command_download.py 代码修改

# 原本的 command_download.py 代码：

base prompt 为：“我这个 test case 缺少 assert 语句，请你帮我自动填充。”

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
import hashlib
import os
import unittest

import onlinejudge_command.subcommand.download as subcommand_download
import tests.utils
from onlinejudge_command.main import get_parser


def get_files_from_json(samples):
    files = {}
    for i, sample in enumerate(samples):
        for ext in ('in', 'out'):
            if 'name' in sample:
                name = sample['name'] + '.' + ext
            else:
                name = 'sample-{}.{}'.format(i + 1, ext)
            files[name] = hashlib.md5(sample[ext + 'put'].encode()).hexdigest()
    return files


def snippet_call_download(self, url, files, is_system=False, is_silent=False, type='files'):
    assert type in ('files', 'json')
    if type == 'json':
        files = get_files_from_json(files)

    with tests.utils.sandbox([]):
        args = ['download', url]
        if is_system:
            args += ['--system']
        if is_silent:
            args += ['--silent']
        tests.utils.run(args, check=True)
        result = {}
        if os.path.exists('test'):
            for name in os.listdir('test'):
                with open(os.path.join('test', name)) as fh:
                    result[name] = hashlib.md5(fh.buffer.read()).hexdigest()
        self.assertEqual(files, result)


def snippet_call_download_failure(self, url, is_system=False, is_silent=False):
    args = ["download", url]
    if is_system:
        args.append("--system")
    if is_silent:
        args.append("--silent")
    args = get_parser().parse_args(args=args)
    self.assertFalse(subcommand_download.run(args))


def snippet_call_download_twice(self, url1, url2, files, is_system=False, is_silent=False, type='files'):
    assert type in ('files', 'json')
    if type == 'json':
        files = get_files_from_json(files)

    with tests.utils.sandbox([]):
        args = ['download', url1]
        if is_system:
            args += ['--system']
        if is_silent:
            args += ['--silent']
        args = get_parser().parse_args(args=args)
        self.assertTrue(subcommand_download.run(args))

        args = ['download', url2]
        if is_system:
            args += ['--system']
        if is_silent:
            args += ['--silent']
        args = get_parser().parse_args(args=args)
        # download from url2 should be aborted.
        self.assertFalse(subcommand_download.run(args))

        # check download from url1 is not overwritten
        result = {}
        if os.path.exists('test'):
            for name in os.listdir('test'):
                with open(os.path.join('test', name)) as fh:
                    result[name] = hashlib.md5(fh.buffer.read()).hexdigest()
        self.assertEqual(files, result)


class DownloadTest(unittest.TestCase):
    """DownloadTest is a class to test `download` subcommand itself. Don't try to test sample parsers.
    """
    def snippet_call_download(self, *args, **kwargs):
        tests.command_download.snippet_call_download(self, *args, **kwargs)

    def snippet_call_download_failure(self, *args, **kwargs):
        tests.command_download.snippet_call_download_failure(self, *args, **kwargs)

    def test_call_download_atcoder_abc114_c(self):
        self.snippet_call_download('https://atcoder.jp/contests/abc114/tasks/abc114_c', [
            {
                "input": "575\n",
                "output": "4\n"
            },
            {
                "input": "3600\n",
                "output": "13\n"
            },
            {
                "input": "999999999\n",
                "output": "26484\n"
            },
        ], type='json')

    def test_call_download_atcoder_abc003_4(self):
        self.snippet_call_download('https://atcoder.jp/contests/abc003/tasks/abc003_4', [
            {
                "input": "3 2\n2 2\n2 2\n",
                "output": "12\n"
            },
            {
                "input": "4 5\n3 1\n3 0\n",
                "output": "10\n"
            },
            {
                "input": "23 18\n15 13\n100 95\n",
                "output": "364527243\n"
            },
            {
                "input": "30 30\n24 22\n145 132\n",
                "output": "976668549\n"
            },
        ], type='json')

    def test_call_download_invalid_url(self):
        self.snippet_call_download_failure('http://abc001.contest.atcoder.jp/tasks/abc001_100')

    def test_call_download_413(self):
        # This task is not supported.
        self.snippet_call_download_failure('https://chokudai001.contest.atcoder.jp/tasks/chokudai_001_a')


class DownloadInvalidTest(unittest.TestCase):
    def snippet_call_download_failure(self, *args, **kwargs):
        tests.command_download.snippet_call_download_failure(self, *args, **kwargs)

    def snippet_call_download_twice(self, *args, **kwargs):
        tests.command_download.snippet_call_download_twice(self, *args, **kwargs)

    def test_call_download_invalid(self):
        self.snippet_call_download_failure('https://not_exist_contest.jp/tasks/001_a')

    def test_call_download_no_sample_found(self):
        self.snippet_call_download_failure('https://atcoder.jp/contests/tenka1-2013-quala/tasks/tenka1_2013_qualA_a')
        self.snippet_call_download_failure('https://open.kattis.com/problems/hello')

    def test_call_download_twice(self):
        self.snippet_call_download_twice('https://atcoder.jp/contests/abc114/tasks/abc114_c', 'https://atcoder.jp/contests/abc003/tasks/abc003_4', [
            {
                "input": "575\n",
                "output": "4\n"
            },
            {
                "input": "3600\n",
                "output": "13\n"
            },
            {
                "input": "999999999\n",
                "output": "26484\n"
            },
        ], type='json')

# 移除 assert 断言后的 command_download.py 代码

移除 assert 断言和部分提示后，我们发送给 LLM 的代码如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
import hashlib
import os
import unittest

import onlinejudge_command.subcommand.download as subcommand_download
import tests.utils
from onlinejudge_command.main import get_parser


def get_files_from_json(samples):
    files = {}
    for i, sample in enumerate(samples):
        for ext in ('in', 'out'):
            if 'name' in sample:
                name = sample['name'] + '.' + ext
            else:
                name = 'sample-{}.{}'.format(i + 1, ext)
            files[name] = hashlib.md5(sample[ext + 'put'].encode()).hexdigest()
    return files


def snippet_call_download(self, url, files, is_system=False, is_silent=False, type='files'):
    if type == 'json':
        files = get_files_from_json(files)

    with tests.utils.sandbox([]):
        args = ['download', url]
        if is_system:
            args += ['--system']
        if is_silent:
            args += ['--silent']
        tests.utils.run(args, check=True)
        result = {}
        if os.path.exists('test'):
            for name in os.listdir('test'):
                with open(os.path.join('test', name)) as fh:
                    result[name] = hashlib.md5(fh.buffer.read()).hexdigest()


def snippet_call_download_failure(self, url, is_system=False, is_silent=False):
    args = ["download", url]
    if is_system:
        args.append("--system")
    if is_silent:
        args.append("--silent")


def snippet_call_download_twice(self, url1, url2, files, is_system=False, is_silent=False, type='files'):
    if type == 'json':
        files = get_files_from_json(files)

    with tests.utils.sandbox([]):
        args = ['download', url1]
        if is_system:
            args += ['--system']
        if is_silent:
            args += ['--silent']

        args = ['download', url2]
        if is_system:
            args += ['--system']
        if is_silent:
            args += ['--silent']

        result = {}
        if os.path.exists('test'):
            for name in os.listdir('test'):
                with open(os.path.join('test', name)) as fh:
                    result[name] = hashlib.md5(fh.buffer.read()).hexdigest()


class DownloadTest(unittest.TestCase):
    """DownloadTest is a class to test `download` subcommand itself. Don't try to test sample parsers.
    """
    def snippet_call_download(self, *args, **kwargs):
        tests.command_download.snippet_call_download(self, *args, **kwargs)

    def snippet_call_download_failure(self, *args, **kwargs):
        tests.command_download.snippet_call_download_failure(self, *args, **kwargs)

    def test_call_download_atcoder_abc114_c(self):
        self.snippet_call_download('https://atcoder.jp/contests/abc114/tasks/abc114_c', [
            {
                "input": "575\n",
                "output": "4\n"
            },
            {
                "input": "3600\n",
                "output": "13\n"
            },
            {
                "input": "999999999\n",
                "output": "26484\n"
            },
        ], type='json')

    def test_call_download_atcoder_abc003_4(self):
        self.snippet_call_download('https://atcoder.jp/contests/abc003/tasks/abc003_4', [
            {
                "input": "3 2\n2 2\n2 2\n",
                "output": "12\n"
            },
            {
                "input": "4 5\n3 1\n3 0\n",
                "output": "10\n"
            },
            {
                "input": "23 18\n15 13\n100 95\n",
                "output": "364527243\n"
            },
            {
                "input": "30 30\n24 22\n145 132\n",
                "output": "976668549\n"
            },
        ], type='json')

    def test_call_download_invalid_url(self):
        self.snippet_call_download_failure('http://abc001.contest.atcoder.jp/tasks/abc001_100')

    def test_call_download_413(self):
        # This task is not supported.
        self.snippet_call_download_failure('https://chokudai001.contest.atcoder.jp/tasks/chokudai_001_a')


class DownloadInvalidTest(unittest.TestCase):
    def snippet_call_download_failure(self, *args, **kwargs):
        tests.command_download.snippet_call_download_failure(self, *args, **kwargs)

    def snippet_call_download_twice(self, *args, **kwargs):
        tests.command_download.snippet_call_download_twice(self, *args, **kwargs)

    def test_call_download_invalid(self):
        self.snippet_call_download_failure('https://not_exist_contest.jp/tasks/001_a')

    def test_call_download_no_sample_found(self):
        self.snippet_call_download_failure('https://atcoder.jp/contests/tenka1-2013-quala/tasks/tenka1_2013_qualA_a')
        self.snippet_call_download_failure('https://open.kattis.com/problems/hello')

    def test_call_download_twice(self):
        self.snippet_call_download_twice('https://atcoder.jp/contests/abc114/tasks/abc114_c', 'https://atcoder.jp/contests/abc003/tasks/abc003_4', [
            {
                "input": "575\n",
                "output": "4\n"
            },
            {
                "input": "3600\n",
                "output": "13\n"
            },
            {
                "input": "999999999\n",
                "output": "26484\n"
            },
        ], type='json')

# LLM 自动填充 assert 语句后的代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
import hashlib
import os
import unittest

import onlinejudge_command.subcommand.download as subcommand_download
import tests.utils
from onlinejudge_command.main import get_parser


def get_files_from_json(samples):
    files = {}
    for i, sample in enumerate(samples):
        for ext in ('in', 'out'):
            if 'name' in sample:
                name = sample['name'] + '.' + ext
            else:
                name = 'sample-{}.{}'.format(i + 1, ext)
            files[name] = hashlib.md5(sample[ext + 'put'].encode()).hexdigest()
    return files


def snippet_call_download(self, url, files, is_system=False, is_silent=False, type='files'):
    if type == 'json':
        files = get_files_from_json(files)

    with tests.utils.sandbox([]):
        args = ['download', url]
        if is_system:
            args += ['--system']
        if is_silent:
            args += ['--silent']
        tests.utils.run(args, check=True)
        result = {}
        if os.path.exists('test'):
            for name in os.listdir('test'):
                with open(os.path.join('test', name)) as fh:
                    result[name] = hashlib.md5(fh.buffer.read()).hexdigest()
        self.assertEqual(result, files)


def snippet_call_download_failure(self, url, is_system=False, is_silent=False):
    args = ["download", url]
    if is_system:
        args.append("--system")
    if is_silent:
        args.append("--silent")
    with self.assertRaises(SystemExit):
        tests.utils.run(args, check=True)


def snippet_call_download_twice(self, url1, url2, files, is_system=False, is_silent=False, type='files'):
    if type == 'json':
        files = get_files_from_json(files)

    with tests.utils.sandbox([]):
        args = ['download', url1]
        if is_system:
            args += ['--system']
        if is_silent:
            args += ['--silent']
        tests.utils.run(args, check=True)

        args = ['download', url2]
        if is_system:
            args += ['--system']
        if is_silent:
            args += ['--silent']
        tests.utils.run(args, check=True)

        result = {}
        if os.path.exists('test'):
            for name in os.listdir('test'):
                with open(os.path.join('test', name)) as fh:
                    result[name] = hashlib.md5(fh.buffer.read()).hexdigest()
        self.assertEqual(result, files)


class DownloadTest(unittest.TestCase):
    """DownloadTest is a class to test `download` subcommand itself. Don't try to test sample parsers.
    """
    def snippet_call_download(self, *args, **kwargs):
        tests.command_download.snippet_call_download(self, *args, **kwargs)

    def snippet_call_download_failure(self, *args, **kwargs):
        tests.command_download.snippet_call_download_failure(self, *args, **kwargs)

    def test_call_download_atcoder_abc114_c(self):
        self.snippet_call_download('https://atcoder.jp/contests/abc114/tasks/abc114_c', [
            {
                "input": "575\n",
                "output": "4\n"
            },
            {
                "input": "3600\n",
                "output": "13\n"
            },
            {
                "input": "999999999\n",
                "output": "26484\n"
            },
        ], type='json')

    def test_call_download_atcoder_abc003_4(self):
        self.snippet_call_download('https://atcoder.jp/contests/abc003/tasks/abc003_4', [
            {
                "input": "3 2\n2 2\n2 2\n",
                "output": "12\n"
            },
            {
                "input": "4 5\n3 1\n3 0\n",
                "output": "10\n"
            },
            {
                "input": "23 18\n15 13\n100 95\n",
                "output": "364527243\n"
            },
            {
                "input": "30 30\n24 22\n145 132\n",
                "output": "976668549\n"
            },
        ], type='json')

    def test_call_download_invalid_url(self):
        self.snippet_call_download_failure('http://abc001.contest.atcoder.jp/tasks/abc001_100')

    def test_call_download_413(self):
        # This task is not supported.
        self.snippet_call_download_failure('https://chokudai001.contest.atcoder.jp/tasks/chokudai_001_a')


class DownloadInvalidTest(unittest.TestCase):
    def snippet_call_download_failure(self, *args, **kwargs):
        tests.command_download.snippet_call_download_failure(self, *args, **kwargs)

    def snippet_call_download_twice(self, *args, **kwargs):
        tests.command_download.snippet_call_download_twice(self, *args, **kwargs)

    def test_call_download_invalid(self):
        self.snippet_call_download_failure('https://not_exist_contest.jp/tasks/001_a')

    def test_call_download_no_sample_found(self):
        self.snippet_call_download_failure('https://atcoder.jp/contests/tenka1-2013-quala/tasks/tenka1_2013_qualA_a')
        self.snippet_call_download_failure('https://open.kattis.com/problems/hello')

    def test_call_download_twice(self):
        self.snippet_call_download_twice('https://atcoder.jp/contests/abc114/tasks/abc114_c', 'https://atcoder.jp/contests/abc003/tasks/abc003_4', [
            {
                "input": "575\n",
                "output": "4\n"
            },
            {
                "input": "3600\n",
                "output": "13\n"
            },
            {
                "input": "999999999\n",
                "output": "26484\n"
            },
        ], type='json')