/* * Copyright (C) 2017 Samsung Electronics Co., Ltd. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. */ #include #include #include #include #include #include #include #include #include #include #include "internal.h" #ifdef CONFIG_CRYPTO_DISKCIPHER_DEBUG #include #include #include #include #define DUMP_MAX 20 struct dump_err { struct page *page; struct bio bio; struct fmp_crypto_info ci; enum diskcipher_dbg api; }; struct diskc_debug_info { struct dump_err dump[DUMP_MAX]; int err; int cnt[DISKC_USER_MAX][2]; }; static struct diskc_debug_info diskc_dbg; void crypto_diskcipher_debug(enum diskcipher_dbg api, int bi_opf) { int idx = 0; struct diskc_debug_info *dbg = &diskc_dbg; if (api <= DISKC_API_MAX) dbg->cnt[api][bi_opf]++; else { if (bi_opf) idx = 1; dbg->cnt[api][idx]++; } } static void print_err(void) { int i, j; struct bio_vec *bv; /* bio page list */ struct bio *bio; struct fmp_crypto_info *ci; struct diskc_debug_info *dbg = &diskc_dbg; for (j = 0; j < dbg->err; j++) { bio = &dbg->dump[j].bio; ci = &dbg->dump[j].ci; if (bio) { #ifdef CONFIG_BLK_DEV_CRYPT pr_info ("%s(%d/%d): bio:%p ci:%p page:%p flag:%x, opf:%x, crypt:%p\n", __func__, j, dbg->err, bio, ci, &dbg->dump[j].page, bio->bi_flags, bio->bi_opf, bio->bi_cryptd); #else pr_info ("%s(%d/%d): bio:%p ci:%p page:%p flag:%x, opf:%x\n", __func__, j, dbg->err, bio, ci, &dbg->dump[j].page, bio->bi_flags, bio->bi_opf); #endif print_hex_dump(KERN_CONT, "bio:", DUMP_PREFIX_OFFSET, 16, 1, bio, sizeof(struct bio), false); for (i = 0; i < bio->bi_max_vecs; i++) { bv = &bio->bi_io_vec[i]; pr_info("bv[%d] page:%p len:%d offset:%d\n", i, bv->bv_page, bv->bv_len, bv->bv_offset); } } if (ci) { pr_info("[ci] key_size:%d algo_mode:%d\n", ci->key_size, ci->algo_mode); print_hex_dump(KERN_CONT, "key:", DUMP_PREFIX_OFFSET, 16, 1, ci->key, sizeof(ci->key), false); } } } static void disckipher_log_show(struct seq_file *m) { int i; struct diskc_debug_info *dbg = &diskc_dbg; char name[DISKC_USER_MAX][32] = { "ALLOC", "FREE", "FREEREQ", "SETKEY", "SET", "GET", "CRYPT", "CLEAR", "DISKC_API_MAX", "FS_PAGEIO", "FS_READP", "FS_DIO", "FS_BLOCK_WRITE", "FS_ZEROPAGE", "BLK_BH", "DMCRYPT", "DISKC_MERGE", "DISKC_MERGE_ERR_INODE", "DISKC_MERGE_ERR_DISK", "FS_DEC_WARN", "FS_ENC_WARN", "DISKC_MERGE_DIO", "DISKC_FREE_REQ_WARN", "DISKC_FREE_WQ_WARN", "DISKC_CRYPT_WARN", "DM_CRYPT_NONENCRYPT", "DM_CRYPT_CTR", "DM_CRYPT_DTR", "DM_CRYPT_OVER", "F2FS_gc", "F2FS_gc_data_page", "F2FS_gc_data_page_no_key", "F2FS_gc_data_page_no_key_FC", "F2FS_gc_data_page_FC", "F2FS_gc_data_block", "F2FS_gc_data_block_key", "F2FS_gc_data_block_err1", "F2FS_gc_data_block_err2", "F2FS_gc_data_block_err3", "F2FS_gc_skip", "DISKC_ERR", "DISKC_NO_KEY_ERR", "DISKC_NO_SYNC_ERR", "DISKC_NO_CRYPT_ERR", "DISKC_NO_DISKC_ERR"}; for (i = 0; i < DISKC_USER_MAX; i++) if (dbg->cnt[i][0] || dbg->cnt[i][1]) seq_printf(m, "%s\t: %6u(err:%u)\n", name[i], dbg->cnt[i][0], dbg->cnt[i][1]); if (dbg->err) print_err(); } /* check diskcipher for FBE */ #undef DISKC_FS_ENCRYPT_DEBUG /* it's not work on fips */ #ifdef DISKC_FS_ENCRYPT_DEBUG static void dump_err(struct crypto_diskcipher *ci, enum diskcipher_dbg api, struct bio *bio, struct page *page) { struct diskc_debug_info *dbg = &diskc_dbg; if ((dbg->err < DUMP_MAX) && ci) { struct crypto_tfm *tfm = crypto_diskcipher_tfm(ci); dbg->dump[dbg->err].api = api; memcpy(&dbg->dump[dbg->err].ci, crypto_tfm_ctx(tfm), sizeof(struct fmp_crypto_info)); if (page) dbg->dump[dbg->err].page = page; if (bio) memcpy(&dbg->dump[dbg->err].bio, bio, sizeof(struct bio)); } dbg->err++; } static bool crypto_diskcipher_check(struct bio *bio) { int ret = 0; struct crypto_diskcipher *ci = NULL; struct inode *inode = NULL; struct page *page = NULL; if (!bio) { pr_err("%s: doesn't exist bio\n", __func__); goto out; } page = bio->bi_io_vec[0].bv_page; if (page && !PageAnon(page) && bio) if (page->mapping) if (page->mapping->host) { if (page->mapping->host->i_crypt_info) { inode = page->mapping->host; ci = fscrypt_get_bio_cryptd(inode); if (ci && (bio->bi_cryptd != ci) && (!(bio->bi_flags & REQ_OP_DISCARD))) { pr_err("%s: no sync err\n", __func__); dump_err(ci, DISKC_API_GET, bio, page); crypto_diskcipher_debug(DISKC_NO_SYNC_ERR, 0); ret = -EINVAL; } if (!ci) { crypto_diskcipher_debug(DISKC_NO_DISKC_ERR, 1); pr_err("%s: no crypt err\n", __func__); ret = -EINVAL; } } else { crypto_diskcipher_debug(DISKC_NO_KEY_ERR, 1); ret = -EINVAL; } } out: crypto_diskcipher_debug(DISKC_API_GET, ret); return ret; } #else #define crypto_diskcipher_check(a) (0) #endif #else #define crypto_diskcipher_check(a) (0) #define disckipher_log_show(a) do { } while (0) #endif struct crypto_diskcipher *crypto_diskcipher_get(struct bio *bio) { if (!bio || !virt_addr_valid(bio)) { pr_err("%s: Invalid bio:%p\n", __func__, bio); return NULL; } if (bio->bi_opf & REQ_CRYPT) { if (bio->bi_cryptd) { if (!crypto_diskcipher_check(bio)) return bio->bi_cryptd; else return ERR_PTR(-EINVAL); } else { crypto_diskcipher_debug(DISKC_NO_CRYPT_ERR, 0); return ERR_PTR(-EINVAL); } } return NULL; } void crypto_diskcipher_set(struct bio *bio, struct crypto_diskcipher *tfm, u64 dun) { if (bio && tfm) { bio->bi_opf |= REQ_CRYPT; bio->bi_cryptd = tfm; #ifdef CONFIG_CRYPTO_DISKCIPHER_DUN if (dun) bio->bi_iter.bi_dun = dun; #endif } crypto_diskcipher_debug(DISKC_API_SET, 0); } /* debug freerq */ enum diskc_status { DISKC_ST_INIT, DISKC_ST_FREE_REQ, DISKC_ST_FREE, }; int crypto_diskcipher_setkey(struct crypto_diskcipher *tfm, const char *in_key, unsigned int key_len, bool persistent) { struct crypto_tfm *base = crypto_diskcipher_tfm(tfm); struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg); if (!cra) { pr_err("%s: doesn't exist cra. base:%p", __func__, base); return -EINVAL; } crypto_diskcipher_debug(DISKC_API_SETKEY, 0); return cra->setkey(base, in_key, key_len, persistent); } int crypto_diskcipher_clearkey(struct crypto_diskcipher *tfm) { struct crypto_tfm *base = crypto_diskcipher_tfm(tfm); struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg); if (!cra) { pr_err("%s: doesn't exist cra. base:%p", __func__, base); return -EINVAL; } return cra->clearkey(base); } int crypto_diskcipher_set_crypt(struct crypto_diskcipher *tfm, void *req) { int ret = 0; struct crypto_tfm *base = crypto_diskcipher_tfm(tfm); struct diskcipher_alg *cra = NULL; if (!base) { pr_err("%s: doesn't exist cra. base:%p", __func__, base); ret = -EINVAL; goto out; } cra = crypto_diskcipher_alg(base->__crt_alg); if (!cra) { pr_err("%s: doesn't exist cra. base:%p\n", __func__, base); ret = -EINVAL; goto out; } if (atomic_read(&tfm->status) == DISKC_ST_FREE) { pr_err("%s: tfm is free\n", __func__); crypto_diskcipher_debug(DISKC_CRYPT_WARN, 0); return -EINVAL; } ret = cra->crypt(base, req); #ifdef USE_FREE_REQ if (!list_empty(&cra->freectrl.freelist)) { if (!atomic_read(&cra->freectrl.freewq_active)) { atomic_set(&cra->freectrl.freewq_active, 1); schedule_delayed_work(&cra->freectrl.freewq, 0); } } #endif out: if (ret) pr_err("%s fails ret:%d, cra:%p\n", __func__, ret, cra); crypto_diskcipher_debug(DISKC_API_CRYPT, ret); return ret; } int crypto_diskcipher_clear_crypt(struct crypto_diskcipher *tfm, void *req) { int ret = 0; struct crypto_tfm *base = crypto_diskcipher_tfm(tfm); struct diskcipher_alg *cra = NULL; if (!base) { pr_err("%s: doesn't exist base, tfm:%p\n", __func__, tfm); ret = -EINVAL; goto out; } cra = crypto_diskcipher_alg(base->__crt_alg); if (!cra) { pr_err("%s: doesn't exist cra. base:%p\n", __func__, base); ret = -EINVAL; goto out; } if (atomic_read(&tfm->status) == DISKC_ST_FREE) { pr_warn("%s: tfm is free\n", __func__); return -EINVAL; } ret = cra->clear(base, req); if (ret) pr_err("%s fails", __func__); out: crypto_diskcipher_debug(DISKC_API_CLEAR, ret); return ret; } #ifndef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS int diskcipher_do_crypt(struct crypto_diskcipher *tfm, struct diskcipher_test_request *req) { int ret; struct crypto_tfm *base = crypto_diskcipher_tfm(tfm); struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg); if (!cra) { pr_err("%s: doesn't exist cra. base:%p\n", __func__, base); ret = -EINVAL; goto out; } if (cra->do_crypt) ret = cra->do_crypt(base, req); else ret = -EINVAL; if (ret) pr_err("%s fails ret:%d", __func__, ret); out: return ret; } #endif static int crypto_diskcipher_init_tfm(struct crypto_tfm *base) { struct crypto_diskcipher *tfm = __crypto_diskcipher_cast(base); atomic_set(&tfm->status, DISKC_ST_INIT); return 0; } #ifdef USE_FREE_REQ static void free_workq_func(struct work_struct *work) { struct diskcipher_alg *cra = container_of(work, struct diskcipher_alg, freectrl.freewq.work); struct diskcipher_freectrl *fctrl = &cra->freectrl; struct crypto_diskcipher *_tfm, *tmp; unsigned long cur_jiffies = jiffies; struct list_head poss_free_list; unsigned long flags; INIT_LIST_HEAD(&poss_free_list); /* pickup freelist */ spin_lock_irqsave(&fctrl->freelist_lock, flags); list_for_each_entry_safe(_tfm, tmp, &fctrl->freelist, node) { if (jiffies_to_msecs(cur_jiffies - _tfm->req_jiffies) > fctrl->max_io_ms) list_move_tail(&_tfm->node, &poss_free_list); } spin_unlock_irqrestore(&fctrl->freelist_lock, flags); list_for_each_entry_safe(_tfm, tmp, &poss_free_list, node) { if (atomic_read (&_tfm->status) != DISKC_ST_FREE_REQ) crypto_diskcipher_debug(DISKC_FREE_WQ_WARN, 0); crypto_free_diskcipher(_tfm); } if (!list_empty(&fctrl->freelist)) schedule_delayed_work(&fctrl->freewq, msecs_to_jiffies(fctrl->max_io_ms)); else atomic_set(&fctrl->freewq_active, 0); } #endif void crypto_free_req_diskcipher(struct crypto_diskcipher *tfm) { #ifdef USE_FREE_REQ struct crypto_tfm *base = crypto_diskcipher_tfm(tfm); struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg); struct diskcipher_freectrl *fctrl = &cra->freectrl; unsigned long flags; if (atomic_read(&tfm->status) != DISKC_ST_INIT) { crypto_diskcipher_debug(DISKC_FREE_REQ_WARN, 0); pr_warn("%s: already submit status:%d\n", __func__, atomic_read(&tfm->status)); return; } atomic_set(&tfm->status, DISKC_ST_FREE_REQ); INIT_LIST_HEAD(&tfm->node); tfm->req_jiffies = jiffies; spin_lock_irqsave(&fctrl->freelist_lock, flags); list_move_tail(&tfm->node, &fctrl->freelist); spin_unlock_irqrestore(&fctrl->freelist_lock, flags); crypto_diskcipher_debug(DISKC_API_FREEREQ, 0); #else crypto_free_diskcipher(tfm); #endif } unsigned int crypto_diskcipher_extsize(struct crypto_alg *alg) { return alg->cra_ctxsize + (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1)); } static void crypto_diskcipher_show(struct seq_file *m, struct crypto_alg *alg) { seq_printf(m, "type : diskcipher\n"); disckipher_log_show(m); } static const struct crypto_type crypto_diskcipher_type = { .extsize = crypto_diskcipher_extsize, .init_tfm = crypto_diskcipher_init_tfm, #ifdef CONFIG_PROC_FS .show = crypto_diskcipher_show, #endif .maskclear = ~CRYPTO_ALG_TYPE_MASK, .maskset = CRYPTO_ALG_TYPE_MASK, .type = CRYPTO_ALG_TYPE_DISKCIPHER, .tfmsize = offsetof(struct crypto_diskcipher, base), }; #define DISKC_NAME "-disk" #define DISKC_NAME_SIZE (5) #define DISKCIPHER_MAX_IO_MS (1000) struct crypto_diskcipher *crypto_alloc_diskcipher(const char *alg_name, u32 type, u32 mask, bool force) { crypto_diskcipher_debug(DISKC_API_ALLOC, 0); if (force) { if (strlen(alg_name) + DISKC_NAME_SIZE < CRYPTO_MAX_ALG_NAME) { char diskc_name[CRYPTO_MAX_ALG_NAME]; strcpy(diskc_name, alg_name); strcat(diskc_name, DISKC_NAME); return crypto_alloc_tfm(diskc_name, &crypto_diskcipher_type, type, mask); } } else { return crypto_alloc_tfm(alg_name, &crypto_diskcipher_type, type, mask); } return NULL; } void crypto_free_diskcipher(struct crypto_diskcipher *tfm) { crypto_diskcipher_debug(DISKC_API_FREE, 0); atomic_set(&tfm->status, DISKC_ST_FREE); crypto_destroy_tfm(tfm, crypto_diskcipher_tfm(tfm)); } int crypto_register_diskcipher(struct diskcipher_alg *alg) { struct crypto_alg *base = &alg->base; #ifdef USE_FREE_REQ struct diskcipher_freectrl *fctrl = &alg->freectrl; INIT_LIST_HEAD(&fctrl->freelist); INIT_DELAYED_WORK(&fctrl->freewq, free_workq_func); spin_lock_init(&fctrl->freelist_lock); if (!fctrl->max_io_ms) fctrl->max_io_ms = DISKCIPHER_MAX_IO_MS; #endif base->cra_type = &crypto_diskcipher_type; base->cra_flags = CRYPTO_ALG_TYPE_DISKCIPHER; return crypto_register_alg(base); } void crypto_unregister_diskcipher(struct diskcipher_alg *alg) { crypto_unregister_alg(&alg->base); } int crypto_register_diskciphers(struct diskcipher_alg *algs, int count) { int i, ret; for (i = 0; i < count; i++) { ret = crypto_register_diskcipher(algs + i); if (ret) goto err; } return 0; err: for (--i; i >= 0; --i) crypto_unregister_diskcipher(algs + i); return ret; } void crypto_unregister_diskciphers(struct diskcipher_alg *algs, int count) { int i; for (i = count - 1; i >= 0; --i) crypto_unregister_diskcipher(algs + i); }