Stowage/clamav
2
0
Fork 0
mirror of https://github.com/Cisco-Talos/clamav.git synced 2025-10-19 10:23:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Fix YARA arena management, improve error reporting, clean up some code.

Browse source
This commit is contained in:
Steven Morgan 2015-05-07 15:50:37 -04:00
parent 0abc761f12
commit d03c18bed3
13 changed files with 255 additions and 293 deletions
Download patch file Download diff file Expand all files Collapse all files

40
libclamav/others.c
View file

@ -315,7 +315,7 @@ int cl_init(unsigned int initoptions)
struct cl_engine *cl_engine_new(void)
{
struct cl_engine *new;
struct cl_engine *new;
cli_intel_t *intel;
new = (struct cl_engine *) cli_calloc(1, sizeof(struct cl_engine));
@ -436,6 +436,44 @@ struct cl_engine *cl_engine_new(void)
new->pcre_recmatch_limit = CLI_DEFAULT_PCRE_RECMATCH_LIMIT;
new->pcre_max_filesize = CLI_DEFAULT_PCRE_MAX_FILESIZE;
/* Initialize YARA */
if (ERROR_SUCCESS != yr_arena_create(1024, 0, &new->the_arena)) {
cli_errmsg("cli_engine_new: failed to create the YARA arena\n");
mpool_free(new->mempool, new->dconf);
mpool_free(new->mempool, new->root);
#ifdef USE_MPOOL
mpool_destroy(new->mempool);
#endif
free(new);
free(intel);
return NULL;
}
if (ERROR_SUCCESS != yr_hash_table_create(10007, &new->rules_table)) {
cli_errmsg("cli_engine_new: failed to create the YARA rules table\n");
yr_arena_destroy(new->the_arena);
mpool_free(new->mempool, new->dconf);
mpool_free(new->mempool, new->root);
#ifdef USE_MPOOL
mpool_destroy(new->mempool);
#endif
free(new);
free(intel);
return NULL;
}
if (ERROR_SUCCESS != yr_hash_table_create(10007, &new->objects_table)) {
cli_errmsg("cli_engine_new: failed to create the YARA objects table\n");
yr_hash_table_destroy(new->rules_table, NULL);
yr_arena_destroy(new->the_arena);
mpool_free(new->mempool, new->dconf);
mpool_free(new->mempool, new->root);
#ifdef USE_MPOOL
mpool_destroy(new->mempool);
#endif
free(new);
free(intel);
return NULL;
}
cli_dbgmsg("Initialized %s engine\n", cl_retver());
return new;
}

6
libclamav/others.h
View file

@ -53,6 +53,7 @@
#ifdef HAVE_JSON
#include "json.h"
#endif
#include "yara_clam.h"
#if HAVE_LIBXML2
#define CLAMAV_MIN_XMLREADER_FLAGS (XML_PARSE_NOERROR | XML_PARSE_NONET)
@ -356,6 +357,11 @@ struct cl_engine {
uint64_t pcre_match_limit;
uint64_t pcre_recmatch_limit;
uint64_t pcre_max_filesize;
/* YARA */
YR_ARENA * the_arena;
YR_HASH_TABLE * rules_table;
YR_HASH_TABLE * objects_table;
};
struct cl_settings {

46
libclamav/readdb.c
View file

@ -3774,11 +3774,7 @@ static int cli_loadyara(FILE *fs, struct cl_engine *engine, unsigned int *signo,
STAILQ_INIT(&compiler.rule_q);
STAILQ_INIT(&compiler.current_rule_string_q);
rc = yr_hash_table_create(10007, &compiler.rules_table);
if (rc == ERROR_SUCCESS)
rc = yr_hash_table_create(10007, &compiler.objects_table);
if (rc == ERROR_SUCCESS)
rc = yr_arena_create(65536, 0, &compiler.sz_arena);
rc = yr_arena_create(65536, 0, &compiler.sz_arena);
if (rc == ERROR_SUCCESS)
rc = yr_arena_create(65536, 0, &compiler.rules_arena);
if (rc == ERROR_SUCCESS)
@ -3792,17 +3788,18 @@ static int cli_loadyara(FILE *fs, struct cl_engine *engine, unsigned int *signo,
compiler.loop_for_of_mem_offset = -1;
ns.name = "default";
compiler.current_namespace = &ns;
compiler.the_arena = engine->the_arena;
compiler.rules_table = engine->rules_table;
compiler.objects_table = engine->objects_table;
rc = yr_lex_parse_rules_file(fs, &compiler);
if (rc > 0) { /* rc = number of errors */
/* TODO - handle the various errors? */
cli_errmsg("cli_loadyara: failed to parse rules file %s, error count %i\n", dbname, rc);
yr_hash_table_destroy(compiler.rules_table, NULL);
yr_hash_table_destroy(compiler.objects_table, NULL);
// yr_arena_destroy(compiler.sz_arena);
// yr_arena_destroy(compiler.rules_arena);
yr_arena_destroy(compiler.sz_arena);
yr_arena_destroy(compiler.rules_arena);
yr_arena_destroy(compiler.code_arena);
// yr_arena_destroy(compiler.strings_arena);
yr_arena_destroy(compiler.strings_arena);
yr_arena_destroy(compiler.metas_arena);
#ifdef YARA_FINISHED
return CL_EMALFDB;
@ -3826,12 +3823,10 @@ static int cli_loadyara(FILE *fs, struct cl_engine *engine, unsigned int *signo,
}
}
yr_hash_table_destroy(compiler.rules_table, NULL);
yr_hash_table_destroy(compiler.objects_table, NULL);
// yr_arena_destroy(compiler.sz_arena);
// yr_arena_destroy(compiler.rules_arena);
yr_arena_append(engine->the_arena, compiler.sz_arena);
yr_arena_append(engine->the_arena, compiler.rules_arena);
yr_arena_append(engine->the_arena, compiler.strings_arena);
yr_arena_destroy(compiler.code_arena);
// yr_arena_destroy(compiler.strings_arena);
yr_arena_destroy(compiler.metas_arena);
if(rc)
@ -4285,6 +4280,7 @@ int cl_load(const char *path, struct cl_engine *engine, unsigned int *signo, uns
cli_errmsg("cl_load(%s): Not supported database file type\n", path);
return CL_EOPEN;
}
#ifdef YARA_PROTO
if (yara_total) {
cli_yaramsg("$$$$$$$$$$$$ YARA $$$$$$$$$$$$\n");
@ -4553,9 +4549,6 @@ int cl_engine_free(struct cl_engine *engine)
for(j = 0; j < root->ac_lsigs; j++) {
if (root->ac_lsigtable[j]->type == CLI_LSIG_NORMAL)
mpool_free(engine->mempool, root->ac_lsigtable[j]->u.logic);
else if (root->ac_lsigtable[j]->type == CLI_YARA_NORMAL ||
root->ac_lsigtable[j]->type == CLI_YARA_NORMAL)
free(root->ac_lsigtable[j]->u.code_start);
FREE_TDB(root->ac_lsigtable[j]->tdb);
mpool_free(engine->mempool, root->ac_lsigtable[j]);
}
@ -4667,6 +4660,16 @@ int cl_engine_free(struct cl_engine *engine)
#ifdef USE_MPOOL
if(engine->mempool) mpool_destroy(engine->mempool);
#endif
if (engine->rules_table)
yr_hash_table_destroy(engine->rules_table, NULL);
if (engine->objects_table)
yr_hash_table_destroy(engine->objects_table, NULL);
if (engine->the_arena)
yr_arena_destroy(engine->the_arena);
free(engine);
return CL_SUCCESS;
}
@ -4680,6 +4683,13 @@ int cl_engine_compile(struct cl_engine *engine)
if(!engine)
return CL_ENULLARG;
/* Free YARA hash tables - only needed for parse and load */
if (engine->rules_table)
yr_hash_table_destroy(engine->rules_table, NULL);
if (engine->objects_table)
yr_hash_table_destroy(engine->objects_table, NULL);
engine->rules_table = engine->objects_table = NULL;
if(!engine->ftypes)
if((ret = cli_loadftm(NULL, engine, 0, 1, NULL)))
return ret;

6
libclamav/yara_arena.c
View file

@ -31,8 +31,8 @@ from files.
#include <time.h>
#include <stdint.h>
#include "yara_clam.h"
#include <yara_arena.h>
#include "yara_clam.h"
#if REAL_YARA
#include <yara/mem.h>
#include <yara/error.h>
@ -750,7 +750,7 @@ int yr_arena_write_string(
(void**) written_string);
}
#if REAL_YARA
//
// yr_arena_append
//
@ -777,7 +777,7 @@ int yr_arena_append(
return ERROR_SUCCESS;
}
#endif
#if REAL_YARA
//

118
libclamav/yara_clam.h
View file

@ -41,9 +41,9 @@ limitations under the License.
#define _YARA_CLAM_H_
#include "shared/queue.h"
#include "others.h"
#include "yara_arena.h"
#include "yara_hash.h"
#include "others.h"
/* From libyara/include/yara/types.h */
#define DECLARE_REFERENCE(type, name) \
@ -313,95 +313,6 @@ typedef struct _SIZED_STRING
#define RE_FLAGS_DOT_ALL 0x80
#define RE_FLAGS_NOT_AT_START 0x100
/* From libyara/include/yara/exec.h */
#define UNDEFINED 0xFFFABADAFABADAFFLL
#define IS_UNDEFINED(x) ((x) == UNDEFINED)
#define OP_HALT 255
#define OP_AND 1
#define OP_OR 2
#define OP_XOR 3
#define OP_NOT 4
#define OP_LT 5
#define OP_GT 6
#define OP_LE 7
#define OP_GE 8
#define OP_EQ 9
#define OP_NEQ 10
#define OP_SZ_EQ 11
#define OP_SZ_NEQ 12
#define OP_SZ_TO_BOOL 13
#define OP_ADD 14
#define OP_SUB 15
#define OP_MUL 16
#define OP_DIV 17
#define OP_MOD 18
#define OP_NEG 19
#define OP_SHL 20
#define OP_SHR 21
#define OP_PUSH 22
#define OP_POP 23
#define OP_CALL 24
#define OP_OBJ_LOAD 25
#define OP_OBJ_VALUE 26
#define OP_OBJ_FIELD 27
#define OP_INDEX_ARRAY 28
#define OP_STR_COUNT 29
#define OP_STR_FOUND 30
#define OP_STR_FOUND_AT 31
#define OP_STR_FOUND_IN 32
#define OP_STR_OFFSET 33
#define OP_OF 34
#define OP_PUSH_RULE 35
#define OP_MATCH_RULE 36
#define OP_INCR_M 37
#define OP_CLEAR_M 38
#define OP_ADD_M 39
#define OP_POP_M 40
#define OP_PUSH_M 41
#define OP_SWAPUNDEF 42
#define OP_JNUNDEF 43
#define OP_JLE 44
#define OP_FILESIZE 45
#define OP_ENTRYPOINT 46
#define OP_INT8 47
#define OP_INT16 48
#define OP_INT32 49
#define OP_UINT8 50
#define OP_UINT16 51
#define OP_UINT32 52
#define OP_CONTAINS 53
#define OP_MATCHES 54
#define OP_IMPORT 55
/*
typedef struct _YR_MATCH
{
int64_t offset;
int32_t length;
union {
uint8_t* data; // Confirmed matches use "data",
int32_t chain_length; // unconfirmed ones use "chain_length"
};
struct _YR_MATCH* prev;
struct _YR_MATCH* next;
} YR_MATCH;
typedef struct _YR_MATCHES
{
int32_t count;
DECLARE_REFERENCE(YR_MATCH*, head);
DECLARE_REFERENCE(YR_MATCH*, tail);
} YR_MATCHES;
*/
typedef struct _YR_META
{
int32_t type;
@ -564,30 +475,31 @@ typedef struct _yc_compiler {
int last_error_line;
int last_result;
YR_ARENA* sz_arena;
YR_ARENA* rules_arena;
YR_ARENA* strings_arena;
YR_ARENA* code_arena;
YR_ARENA* metas_arena;
YR_HASH_TABLE* rules_table;
YR_HASH_TABLE* objects_table;
YR_NAMESPACE* current_namespace;
yc_string* current_rule_strings;
YR_ARENA *sz_arena;
YR_ARENA *rules_arena;
YR_ARENA *strings_arena;
YR_ARENA *code_arena;
YR_ARENA *metas_arena;
YR_ARENA *the_arena;
YR_HASH_TABLE *rules_table;
YR_HASH_TABLE *objects_table;
YR_NAMESPACE *current_namespace;
yc_string *current_rule_strings;
uint32_t current_rule_flags;
uint32_t current_rule_clflags;
int8_t* loop_address[MAX_LOOP_NESTING];
char* loop_identifier[MAX_LOOP_NESTING];
int8_t *loop_address[MAX_LOOP_NESTING];
char *loop_identifier[MAX_LOOP_NESTING];
int loop_depth;
int loop_for_of_mem_offset;
char last_error_extra_info[MAX_COMPILER_ERROR_EXTRA_INFO];
char lex_buf[LEX_BUF_SIZE];
char* lex_buf_ptr;
char *lex_buf_ptr;
unsigned short lex_buf_len;
char * error_msg;
char *error_msg;
STAILQ_HEAD(rq, _yc_rule) rule_q;
STAILQ_HEAD(cs, _yc_string) current_rule_string_q;

2
libclamav/yara_exec.c
View file

@ -165,7 +165,7 @@ int yr_execute_code(
while(1)
{
// cli_errmsg("yara_exec: executing %i\n", *ip);
cli_dbgmsg("yara_exec: executing %i\n", *ip);
switch(*ip)
{
case OP_HALT:

291
libclamav/yara_grammar.c
View file

File diff suppressed because it is too large Load diff

2
libclamav/yara_grammar.h
View file

@ -151,7 +151,7 @@ typedef union YYSTYPE
{
/* Line 2068 of yacc.c */
#line 213 "yara_grammar.y"
#line 214 "yara_grammar.y"
SIZED_STRING* sized_string;
char* c_string;

1
libclamav/yara_grammar.y
View file

@ -62,6 +62,7 @@ limitations under the License.
#include "libclamav/yara_grammar.h"
#include "libclamav/yara_lexer.h"
#include "libclamav/yara_parser.h"
#include "libclamav/yara_exec.h"
#endif
#define YYERROR_VERBOSE

2
libclamav/yara_hash.c
View file

@ -17,8 +17,8 @@ limitations under the License.
#include <stdint.h>
#include <string.h>
#include "yara_clam.h"
#include <yara_hash.h>
#include "yara_clam.h"
#if REAL_YARA
#include <yara/mem.h>
#include <yara/error.h>

14
libclamav/yara_lexer.c
View file

@ -3039,15 +3039,11 @@ void yyerror(
#else
compiler->errors++;
if (error_message != NULL)
cli_errmsg("yara_lexer:yyerror() %s\n", error_message);
else if (compiler->error_msg != NULL)
cli_errmsg("yara_lexer:yyerror() %s\n", compiler->error_msg);
else if (compiler->last_error_extra_info[0] != (char) 0)
cli_errmsg("yara_lexer:yyerror() %s\n", compiler->last_error_extra_info);
else
cli_errmsg("yara_lexer:yyerror() error unknown\n");
if (compiler->last_result != ERROR_SUCCESS)
cli_errmsg("yara_lexer:yyerror() last result is %i\n", compiler->last_result);
cli_errmsg("yara_lexer:yyerror() error message: %s\n", error_message);
if (compiler->error_msg != NULL)
cli_errmsg("yara_lexer:yyerror() compiler error message: %s\n", compiler->error_msg);
if (compiler->last_error_extra_info[0] != (char) 0)
cli_errmsg("yara_lexer:yyerror() error extra info: %s\n", compiler->last_error_extra_info);
if (compiler->last_result != ERROR_SUCCESS)
cli_errmsg("yara_lexer:yyerror() last result is %i\n", compiler->last_result);
if (compiler->error_line != 0)

14
libclamav/yara_lexer.l
View file

@ -705,15 +705,11 @@ void yyerror(
#else
compiler->errors++;
if (error_message != NULL)
cli_errmsg("yara_lexer:yyerror() %s\n", error_message);
else if (compiler->error_msg != NULL)
cli_errmsg("yara_lexer:yyerror() %s\n", compiler->error_msg);
else if (compiler->last_error_extra_info[0] != (char) 0)
cli_errmsg("yara_lexer:yyerror() %s\n", compiler->last_error_extra_info);
else
cli_errmsg("yara_lexer:yyerror() error unknown\n");
if (compiler->last_result != ERROR_SUCCESS)
cli_errmsg("yara_lexer:yyerror() last result is %i\n", compiler->last_result);
cli_errmsg("yara_lexer:yyerror() error message: %s\n", error_message);
if (compiler->error_msg != NULL)
cli_errmsg("yara_lexer:yyerror() compiler error message: %s\n", compiler->error_msg);
if (compiler->last_error_extra_info[0] != (char) 0)
cli_errmsg("yara_lexer:yyerror() error extra info: %s\n", compiler->last_error_extra_info);
if (compiler->last_result != ERROR_SUCCESS)
cli_errmsg("yara_lexer:yyerror() last result is %i\n", compiler->last_result);
if (compiler->error_line != 0)

6
libclamav/yara_parser.c
View file

@ -55,6 +55,7 @@ limitations under the License.
#include "yara_clam.h"
#include "yara_grammar.h"
#include "yara_lexer.h"
#include "yara_exec.h"
#include "others.h"
#endif
@ -811,8 +812,9 @@ int yr_parser_reduce_rule_declaration(
//Yara condition code will work OK as long as it is less than 64K.
//FAIL_ON_COMPILER_ERROR(yr_arena_coalesce(compiler->code_arena));
rule->code_start = yr_arena_base_address(compiler->code_arena);
compiler->code_arena->page_list_head->address = NULL;
yr_arena_destroy(compiler->code_arena);
yr_arena_append(compiler->the_arena, compiler->code_arena);
// compiler->code_arena->page_list_head->address = NULL;
// yr_arena_destroy(compiler->code_arena);
FAIL_ON_COMPILER_ERROR(yr_arena_create(65536, 0, &compiler->code_arena));
STAILQ_INSERT_TAIL(&compiler->rule_q, rule, link);
#endif