drivers/acpi/nfit.c

   1 /*
   2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #include <linux/list_sort.h>
  14 #include <linux/libnvdimm.h>
  15 #include <linux/module.h>
  16 #include <linux/mutex.h>
  17 #include <linux/ndctl.h>
  18 #include <linux/list.h>
  19 #include <linux/acpi.h>
  20 #include <linux/sort.h>
  21 #include <linux/io.h>
  22 #include "nfit.h"
  23
  24 /*
  25  * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
  26  * irrelevant.
  27  */
  28 #include <asm-generic/io-64-nonatomic-hi-lo.h>
  29
  30 static bool force_enable_dimms;
  31 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
  32 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
  33
  34 static u8 nfit_uuid[NFIT_UUID_MAX][16];
  35
  36 const u8 *to_nfit_uuid(enum nfit_uuids id)
  37 {
  38         return nfit_uuid[id];
  39 }
  40 EXPORT_SYMBOL(to_nfit_uuid);
  41
  42 static struct acpi_nfit_desc *to_acpi_nfit_desc(
  43                 struct nvdimm_bus_descriptor *nd_desc)
  44 {
  45         return container_of(nd_desc, struct acpi_nfit_desc, nd_desc);
  46 }
  47
  48 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
  49 {
  50         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
  51
  52         /*
  53          * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
  54          * acpi_device.
  55          */
  56         if (!nd_desc->provider_name
  57                         || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
  58                 return NULL;
  59
  60         return to_acpi_device(acpi_desc->dev);
  61 }
  62
  63 static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
  64                 struct nvdimm *nvdimm, unsigned int cmd, void *buf,
  65                 unsigned int buf_len)
  66 {
  67         struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
  68         const struct nd_cmd_desc *desc = NULL;
  69         union acpi_object in_obj, in_buf, *out_obj;
  70         struct device *dev = acpi_desc->dev;
  71         const char *cmd_name, *dimm_name;
  72         unsigned long dsm_mask;
  73         acpi_handle handle;
  74         const u8 *uuid;
  75         u32 offset;
  76         int rc, i;
  77
  78         if (nvdimm) {
  79                 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
  80                 struct acpi_device *adev = nfit_mem->adev;
  81
  82                 if (!adev)
  83                         return -ENOTTY;
  84                 dimm_name = nvdimm_name(nvdimm);
  85                 cmd_name = nvdimm_cmd_name(cmd);
  86                 dsm_mask = nfit_mem->dsm_mask;
  87                 desc = nd_cmd_dimm_desc(cmd);
  88                 uuid = to_nfit_uuid(NFIT_DEV_DIMM);
  89                 handle = adev->handle;
  90         } else {
  91                 struct acpi_device *adev = to_acpi_dev(acpi_desc);
  92
  93                 cmd_name = nvdimm_bus_cmd_name(cmd);
  94                 dsm_mask = nd_desc->dsm_mask;
  95                 desc = nd_cmd_bus_desc(cmd);
  96                 uuid = to_nfit_uuid(NFIT_DEV_BUS);
  97                 handle = adev->handle;
  98                 dimm_name = "bus";
  99         }
 100
 101         if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
 102                 return -ENOTTY;
 103
 104         if (!test_bit(cmd, &dsm_mask))
 105                 return -ENOTTY;
 106
 107         in_obj.type = ACPI_TYPE_PACKAGE;
 108         in_obj.package.count = 1;
 109         in_obj.package.elements = &in_buf;
 110         in_buf.type = ACPI_TYPE_BUFFER;
 111         in_buf.buffer.pointer = buf;
 112         in_buf.buffer.length = 0;
 113
 114         /* libnvdimm has already validated the input envelope */
 115         for (i = 0; i < desc->in_num; i++)
 116                 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
 117                                 i, buf);
 118
 119         if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
 120                 dev_dbg(dev, "%s:%s cmd: %s input length: %d\n", __func__,
 121                                 dimm_name, cmd_name, in_buf.buffer.length);
 122                 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
 123                                 4, in_buf.buffer.pointer, min_t(u32, 128,
 124                                         in_buf.buffer.length), true);
 125         }
 126
 127         out_obj = acpi_evaluate_dsm(handle, uuid, 1, cmd, &in_obj);
 128         if (!out_obj) {
 129                 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name,
 130                                 cmd_name);
 131                 return -EINVAL;
 132         }
 133
 134         if (out_obj->package.type != ACPI_TYPE_BUFFER) {
 135                 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n",
 136                                 __func__, dimm_name, cmd_name, out_obj->type);
 137                 rc = -EINVAL;
 138                 goto out;
 139         }
 140
 141         if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
 142                 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__,
 143                                 dimm_name, cmd_name, out_obj->buffer.length);
 144                 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
 145                                 4, out_obj->buffer.pointer, min_t(u32, 128,
 146                                         out_obj->buffer.length), true);
 147         }
 148
 149         for (i = 0, offset = 0; i < desc->out_num; i++) {
 150                 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
 151                                 (u32 *) out_obj->buffer.pointer);
 152
 153                 if (offset + out_size > out_obj->buffer.length) {
 154                         dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n",
 155                                         __func__, dimm_name, cmd_name, i);
 156                         break;
 157                 }
 158
 159                 if (in_buf.buffer.length + offset + out_size > buf_len) {
 160                         dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n",
 161                                         __func__, dimm_name, cmd_name, i);
 162                         rc = -ENXIO;
 163                         goto out;
 164                 }
 165                 memcpy(buf + in_buf.buffer.length + offset,
 166                                 out_obj->buffer.pointer + offset, out_size);
 167                 offset += out_size;
 168         }
 169         if (offset + in_buf.buffer.length < buf_len) {
 170                 if (i >= 1) {
 171                         /*
 172                          * status valid, return the number of bytes left
 173                          * unfilled in the output buffer
 174                          */
 175                         rc = buf_len - offset - in_buf.buffer.length;
 176                 } else {
 177                         dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
 178                                         __func__, dimm_name, cmd_name, buf_len,
 179                                         offset);
 180                         rc = -ENXIO;
 181                 }
 182         } else
 183                 rc = 0;
 184
 185  out:
 186         ACPI_FREE(out_obj);
 187
 188         return rc;
 189 }
 190
 191 static const char *spa_type_name(u16 type)
 192 {
 193         static const char *to_name[] = {
 194                 [NFIT_SPA_VOLATILE] = "volatile",
 195                 [NFIT_SPA_PM] = "pmem",
 196                 [NFIT_SPA_DCR] = "dimm-control-region",
 197                 [NFIT_SPA_BDW] = "block-data-window",
 198                 [NFIT_SPA_VDISK] = "volatile-disk",
 199                 [NFIT_SPA_VCD] = "volatile-cd",
 200                 [NFIT_SPA_PDISK] = "persistent-disk",
 201                 [NFIT_SPA_PCD] = "persistent-cd",
 202
 203         };
 204
 205         if (type > NFIT_SPA_PCD)
 206                 return "unknown";
 207
 208         return to_name[type];
 209 }
 210
 211 static int nfit_spa_type(struct acpi_nfit_system_address *spa)
 212 {
 213         int i;
 214
 215         for (i = 0; i < NFIT_UUID_MAX; i++)
 216                 if (memcmp(to_nfit_uuid(i), spa->range_guid, 16) == 0)
 217                         return i;
 218         return -1;
 219 }
 220
 221 static bool add_spa(struct acpi_nfit_desc *acpi_desc,
 222                 struct acpi_nfit_system_address *spa)
 223 {
 224         struct device *dev = acpi_desc->dev;
 225         struct nfit_spa *nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa),
 226                         GFP_KERNEL);
 227
 228         if (!nfit_spa)
 229                 return false;
 230         INIT_LIST_HEAD(&nfit_spa->list);
 231         nfit_spa->spa = spa;
 232         list_add_tail(&nfit_spa->list, &acpi_desc->spas);
 233         dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__,
 234                         spa->range_index,
 235                         spa_type_name(nfit_spa_type(spa)));
 236         return true;
 237 }
 238
 239 static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
 240                 struct acpi_nfit_memory_map *memdev)
 241 {
 242         struct device *dev = acpi_desc->dev;
 243         struct nfit_memdev *nfit_memdev = devm_kzalloc(dev,
 244                         sizeof(*nfit_memdev), GFP_KERNEL);
 245
 246         if (!nfit_memdev)
 247                 return false;
 248         INIT_LIST_HEAD(&nfit_memdev->list);
 249         nfit_memdev->memdev = memdev;
 250         list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 251         dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n",
 252                         __func__, memdev->device_handle, memdev->range_index,
 253                         memdev->region_index);
 254         return true;
 255 }
 256
 257 static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
 258                 struct acpi_nfit_control_region *dcr)
 259 {
 260         struct device *dev = acpi_desc->dev;
 261         struct nfit_dcr *nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr),
 262                         GFP_KERNEL);
 263
 264         if (!nfit_dcr)
 265                 return false;
 266         INIT_LIST_HEAD(&nfit_dcr->list);
 267         nfit_dcr->dcr = dcr;
 268         list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 269         dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__,
 270                         dcr->region_index, dcr->windows);
 271         return true;
 272 }
 273
 274 static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
 275                 struct acpi_nfit_data_region *bdw)
 276 {
 277         struct device *dev = acpi_desc->dev;
 278         struct nfit_bdw *nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw),
 279                         GFP_KERNEL);
 280
 281         if (!nfit_bdw)
 282                 return false;
 283         INIT_LIST_HEAD(&nfit_bdw->list);
 284         nfit_bdw->bdw = bdw;
 285         list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
 286         dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__,
 287                         bdw->region_index, bdw->windows);
 288         return true;
 289 }
 290
 291 static bool add_idt(struct acpi_nfit_desc *acpi_desc,
 292                 struct acpi_nfit_interleave *idt)
 293 {
 294         struct device *dev = acpi_desc->dev;
 295         struct nfit_idt *nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt),
 296                         GFP_KERNEL);
 297
 298         if (!nfit_idt)
 299                 return false;
 300         INIT_LIST_HEAD(&nfit_idt->list);
 301         nfit_idt->idt = idt;
 302         list_add_tail(&nfit_idt->list, &acpi_desc->idts);
 303         dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__,
 304                         idt->interleave_index, idt->line_count);
 305         return true;
 306 }
 307
 308 static void *add_table(struct acpi_nfit_desc *acpi_desc, void *table,
 309                 const void *end)
 310 {
 311         struct device *dev = acpi_desc->dev;
 312         struct acpi_nfit_header *hdr;
 313         void *err = ERR_PTR(-ENOMEM);
 314
 315         if (table >= end)
 316                 return NULL;
 317
 318         hdr = table;
 319         switch (hdr->type) {
 320         case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
 321                 if (!add_spa(acpi_desc, table))
 322                         return err;
 323                 break;
 324         case ACPI_NFIT_TYPE_MEMORY_MAP:
 325                 if (!add_memdev(acpi_desc, table))
 326                         return err;
 327                 break;
 328         case ACPI_NFIT_TYPE_CONTROL_REGION:
 329                 if (!add_dcr(acpi_desc, table))
 330                         return err;
 331                 break;
 332         case ACPI_NFIT_TYPE_DATA_REGION:
 333                 if (!add_bdw(acpi_desc, table))
 334                         return err;
 335                 break;
 336         case ACPI_NFIT_TYPE_INTERLEAVE:
 337                 if (!add_idt(acpi_desc, table))
 338                         return err;
 339                 break;
 340         case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
 341                 dev_dbg(dev, "%s: flush\n", __func__);
 342                 break;
 343         case ACPI_NFIT_TYPE_SMBIOS:
 344                 dev_dbg(dev, "%s: smbios\n", __func__);
 345                 break;
 346         default:
 347                 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
 348                 break;
 349         }
 350
 351         return table + hdr->length;
 352 }
 353
 354 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
 355                 struct nfit_mem *nfit_mem)
 356 {
 357         u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 358         u16 dcr = nfit_mem->dcr->region_index;
 359         struct nfit_spa *nfit_spa;
 360
 361         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 362                 u16 range_index = nfit_spa->spa->range_index;
 363                 int type = nfit_spa_type(nfit_spa->spa);
 364                 struct nfit_memdev *nfit_memdev;
 365
 366                 if (type != NFIT_SPA_BDW)
 367                         continue;
 368
 369                 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 370                         if (nfit_memdev->memdev->range_index != range_index)
 371                                 continue;
 372                         if (nfit_memdev->memdev->device_handle != device_handle)
 373                                 continue;
 374                         if (nfit_memdev->memdev->region_index != dcr)
 375                                 continue;
 376
 377                         nfit_mem->spa_bdw = nfit_spa->spa;
 378                         return;
 379                 }
 380         }
 381
 382         dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
 383                         nfit_mem->spa_dcr->range_index);
 384         nfit_mem->bdw = NULL;
 385 }
 386
 387 static int nfit_mem_add(struct acpi_nfit_desc *acpi_desc,
 388                 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
 389 {
 390         u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
 391         struct nfit_memdev *nfit_memdev;
 392         struct nfit_dcr *nfit_dcr;
 393         struct nfit_bdw *nfit_bdw;
 394         struct nfit_idt *nfit_idt;
 395         u16 idt_idx, range_index;
 396
 397         list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
 398                 if (nfit_dcr->dcr->region_index != dcr)
 399                         continue;
 400                 nfit_mem->dcr = nfit_dcr->dcr;
 401                 break;
 402         }
 403
 404         if (!nfit_mem->dcr) {
 405                 dev_dbg(acpi_desc->dev, "SPA %d missing:%s%s\n",
 406                                 spa->range_index, __to_nfit_memdev(nfit_mem)
 407                                 ? "" : " MEMDEV", nfit_mem->dcr ? "" : " DCR");
 408                 return -ENODEV;
 409         }
 410
 411         /*
 412          * We've found enough to create an nvdimm, optionally
 413          * find an associated BDW
 414          */
 415         list_add(&nfit_mem->list, &acpi_desc->dimms);
 416
 417         list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
 418                 if (nfit_bdw->bdw->region_index != dcr)
 419                         continue;
 420                 nfit_mem->bdw = nfit_bdw->bdw;
 421                 break;
 422         }
 423
 424         if (!nfit_mem->bdw)
 425                 return 0;
 426
 427         nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
 428
 429         if (!nfit_mem->spa_bdw)
 430                 return 0;
 431
 432         range_index = nfit_mem->spa_bdw->range_index;
 433         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 434                 if (nfit_memdev->memdev->range_index != range_index ||
 435                                 nfit_memdev->memdev->region_index != dcr)
 436                         continue;
 437                 nfit_mem->memdev_bdw = nfit_memdev->memdev;
 438                 idt_idx = nfit_memdev->memdev->interleave_index;
 439                 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
 440                         if (nfit_idt->idt->interleave_index != idt_idx)
 441                                 continue;
 442                         nfit_mem->idt_bdw = nfit_idt->idt;
 443                         break;
 444                 }
 445                 break;
 446         }
 447
 448         return 0;
 449 }
 450
 451 static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc,
 452                 struct acpi_nfit_system_address *spa)
 453 {
 454         struct nfit_mem *nfit_mem, *found;
 455         struct nfit_memdev *nfit_memdev;
 456         int type = nfit_spa_type(spa);
 457         u16 dcr;
 458
 459         switch (type) {
 460         case NFIT_SPA_DCR:
 461         case NFIT_SPA_PM:
 462                 break;
 463         default:
 464                 return 0;
 465         }
 466
 467         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 468                 int rc;
 469
 470                 if (nfit_memdev->memdev->range_index != spa->range_index)
 471                         continue;
 472                 found = NULL;
 473                 dcr = nfit_memdev->memdev->region_index;
 474                 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
 475                         if (__to_nfit_memdev(nfit_mem)->region_index == dcr) {
 476                                 found = nfit_mem;
 477                                 break;
 478                         }
 479
 480                 if (found)
 481                         nfit_mem = found;
 482                 else {
 483                         nfit_mem = devm_kzalloc(acpi_desc->dev,
 484                                         sizeof(*nfit_mem), GFP_KERNEL);
 485                         if (!nfit_mem)
 486                                 return -ENOMEM;
 487                         INIT_LIST_HEAD(&nfit_mem->list);
 488                 }
 489
 490                 if (type == NFIT_SPA_DCR) {
 491                         struct nfit_idt *nfit_idt;
 492                         u16 idt_idx;
 493
 494                         /* multiple dimms may share a SPA when interleaved */
 495                         nfit_mem->spa_dcr = spa;
 496                         nfit_mem->memdev_dcr = nfit_memdev->memdev;
 497                         idt_idx = nfit_memdev->memdev->interleave_index;
 498                         list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
 499                                 if (nfit_idt->idt->interleave_index != idt_idx)
 500                                         continue;
 501                                 nfit_mem->idt_dcr = nfit_idt->idt;
 502                                 break;
 503                         }
 504                 } else {
 505                         /*
 506                          * A single dimm may belong to multiple SPA-PM
 507                          * ranges, record at least one in addition to
 508                          * any SPA-DCR range.
 509                          */
 510                         nfit_mem->memdev_pmem = nfit_memdev->memdev;
 511                 }
 512
 513                 if (found)
 514                         continue;
 515
 516                 rc = nfit_mem_add(acpi_desc, nfit_mem, spa);
 517                 if (rc)
 518                         return rc;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
 525 {
 526         struct nfit_mem *a = container_of(_a, typeof(*a), list);
 527         struct nfit_mem *b = container_of(_b, typeof(*b), list);
 528         u32 handleA, handleB;
 529
 530         handleA = __to_nfit_memdev(a)->device_handle;
 531         handleB = __to_nfit_memdev(b)->device_handle;
 532         if (handleA < handleB)
 533                 return -1;
 534         else if (handleA > handleB)
 535                 return 1;
 536         return 0;
 537 }
 538
 539 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
 540 {
 541         struct nfit_spa *nfit_spa;
 542
 543         /*
 544          * For each SPA-DCR or SPA-PMEM address range find its
 545          * corresponding MEMDEV(s).  From each MEMDEV find the
 546          * corresponding DCR.  Then, if we're operating on a SPA-DCR,
 547          * try to find a SPA-BDW and a corresponding BDW that references
 548          * the DCR.  Throw it all into an nfit_mem object.  Note, that
 549          * BDWs are optional.
 550          */
 551         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 552                 int rc;
 553
 554                 rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa);
 555                 if (rc)
 556                         return rc;
 557         }
 558
 559         list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
 560
 561         return 0;
 562 }
 563
 564 static ssize_t revision_show(struct device *dev,
 565                 struct device_attribute *attr, char *buf)
 566 {
 567         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 568         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
 569         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
 570
 571         return sprintf(buf, "%d\n", acpi_desc->nfit->header.revision);
 572 }
 573 static DEVICE_ATTR_RO(revision);
 574
 575 static struct attribute *acpi_nfit_attributes[] = {
 576         &dev_attr_revision.attr,
 577         NULL,
 578 };
 579
 580 static struct attribute_group acpi_nfit_attribute_group = {
 581         .name = "nfit",
 582         .attrs = acpi_nfit_attributes,
 583 };
 584
 585 const struct attribute_group *acpi_nfit_attribute_groups[] = {
 586         &nvdimm_bus_attribute_group,
 587         &acpi_nfit_attribute_group,
 588         NULL,
 589 };
 590 EXPORT_SYMBOL_GPL(acpi_nfit_attribute_groups);
 591
 592 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
 593 {
 594         struct nvdimm *nvdimm = to_nvdimm(dev);
 595         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 596
 597         return __to_nfit_memdev(nfit_mem);
 598 }
 599
 600 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
 601 {
 602         struct nvdimm *nvdimm = to_nvdimm(dev);
 603         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 604
 605         return nfit_mem->dcr;
 606 }
 607
 608 static ssize_t handle_show(struct device *dev,
 609                 struct device_attribute *attr, char *buf)
 610 {
 611         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
 612
 613         return sprintf(buf, "%#x\n", memdev->device_handle);
 614 }
 615 static DEVICE_ATTR_RO(handle);
 616
 617 static ssize_t phys_id_show(struct device *dev,
 618                 struct device_attribute *attr, char *buf)
 619 {
 620         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
 621
 622         return sprintf(buf, "%#x\n", memdev->physical_id);
 623 }
 624 static DEVICE_ATTR_RO(phys_id);
 625
 626 static ssize_t vendor_show(struct device *dev,
 627                 struct device_attribute *attr, char *buf)
 628 {
 629         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 630
 631         return sprintf(buf, "%#x\n", dcr->vendor_id);
 632 }
 633 static DEVICE_ATTR_RO(vendor);
 634
 635 static ssize_t rev_id_show(struct device *dev,
 636                 struct device_attribute *attr, char *buf)
 637 {
 638         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 639
 640         return sprintf(buf, "%#x\n", dcr->revision_id);
 641 }
 642 static DEVICE_ATTR_RO(rev_id);
 643
 644 static ssize_t device_show(struct device *dev,
 645                 struct device_attribute *attr, char *buf)
 646 {
 647         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 648
 649         return sprintf(buf, "%#x\n", dcr->device_id);
 650 }
 651 static DEVICE_ATTR_RO(device);
 652
 653 static ssize_t format_show(struct device *dev,
 654                 struct device_attribute *attr, char *buf)
 655 {
 656         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 657
 658         return sprintf(buf, "%#x\n", dcr->code);
 659 }
 660 static DEVICE_ATTR_RO(format);
 661
 662 static ssize_t serial_show(struct device *dev,
 663                 struct device_attribute *attr, char *buf)
 664 {
 665         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 666
 667         return sprintf(buf, "%#x\n", dcr->serial_number);
 668 }
 669 static DEVICE_ATTR_RO(serial);
 670
 671 static ssize_t flags_show(struct device *dev,
 672                 struct device_attribute *attr, char *buf)
 673 {
 674         u16 flags = to_nfit_memdev(dev)->flags;
 675
 676         return sprintf(buf, "%s%s%s%s%s\n",
 677                         flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save " : "",
 678                         flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore " : "",
 679                         flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush " : "",
 680                         flags & ACPI_NFIT_MEM_ARMED ? "arm " : "",
 681                         flags & ACPI_NFIT_MEM_HEALTH_OBSERVED ? "smart " : "");
 682 }
 683 static DEVICE_ATTR_RO(flags);
 684
 685 static struct attribute *acpi_nfit_dimm_attributes[] = {
 686         &dev_attr_handle.attr,
 687         &dev_attr_phys_id.attr,
 688         &dev_attr_vendor.attr,
 689         &dev_attr_device.attr,
 690         &dev_attr_format.attr,
 691         &dev_attr_serial.attr,
 692         &dev_attr_rev_id.attr,
 693         &dev_attr_flags.attr,
 694         NULL,
 695 };
 696
 697 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
 698                 struct attribute *a, int n)
 699 {
 700         struct device *dev = container_of(kobj, struct device, kobj);
 701
 702         if (to_nfit_dcr(dev))
 703                 return a->mode;
 704         else
 705                 return 0;
 706 }
 707
 708 static struct attribute_group acpi_nfit_dimm_attribute_group = {
 709         .name = "nfit",
 710         .attrs = acpi_nfit_dimm_attributes,
 711         .is_visible = acpi_nfit_dimm_attr_visible,
 712 };
 713
 714 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
 715         &nvdimm_attribute_group,
 716         &nd_device_attribute_group,
 717         &acpi_nfit_dimm_attribute_group,
 718         NULL,
 719 };
 720
 721 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
 722                 u32 device_handle)
 723 {
 724         struct nfit_mem *nfit_mem;
 725
 726         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
 727                 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
 728                         return nfit_mem->nvdimm;
 729
 730         return NULL;
 731 }
 732
 733 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
 734                 struct nfit_mem *nfit_mem, u32 device_handle)
 735 {
 736         struct acpi_device *adev, *adev_dimm;
 737         struct device *dev = acpi_desc->dev;
 738         const u8 *uuid = to_nfit_uuid(NFIT_DEV_DIMM);
 739         unsigned long long sta;
 740         int i, rc = -ENODEV;
 741         acpi_status status;
 742
 743         nfit_mem->dsm_mask = acpi_desc->dimm_dsm_force_en;
 744         adev = to_acpi_dev(acpi_desc);
 745         if (!adev)
 746                 return 0;
 747
 748         adev_dimm = acpi_find_child_device(adev, device_handle, false);
 749         nfit_mem->adev = adev_dimm;
 750         if (!adev_dimm) {
 751                 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
 752                                 device_handle);
 753                 return force_enable_dimms ? 0 : -ENODEV;
 754         }
 755
 756         status = acpi_evaluate_integer(adev_dimm->handle, "_STA", NULL, &sta);
 757         if (status == AE_NOT_FOUND) {
 758                 dev_dbg(dev, "%s missing _STA, assuming enabled...\n",
 759                                 dev_name(&adev_dimm->dev));
 760                 rc = 0;
 761         } else if (ACPI_FAILURE(status))
 762                 dev_err(dev, "%s failed to retrieve_STA, disabling...\n",
 763                                 dev_name(&adev_dimm->dev));
 764         else if ((sta & ACPI_STA_DEVICE_ENABLED) == 0)
 765                 dev_info(dev, "%s disabled by firmware\n",
 766                                 dev_name(&adev_dimm->dev));
 767         else
 768                 rc = 0;
 769
 770         for (i = ND_CMD_SMART; i <= ND_CMD_VENDOR; i++)
 771                 if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i))
 772                         set_bit(i, &nfit_mem->dsm_mask);
 773
 774         return force_enable_dimms ? 0 : rc;
 775 }
 776
 777 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
 778 {
 779         struct nfit_mem *nfit_mem;
 780         int dimm_count = 0;
 781
 782         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
 783                 struct nvdimm *nvdimm;
 784                 unsigned long flags = 0;
 785                 u32 device_handle;
 786                 u16 mem_flags;
 787                 int rc;
 788
 789                 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 790                 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
 791                 if (nvdimm) {
 792                         /*
 793                          * If for some reason we find multiple DCRs the
 794                          * first one wins
 795                          */
 796                         dev_err(acpi_desc->dev, "duplicate DCR detected: %s\n",
 797                                         nvdimm_name(nvdimm));
 798                         continue;
 799                 }
 800
 801                 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
 802                         flags |= NDD_ALIASING;
 803
 804                 mem_flags = __to_nfit_memdev(nfit_mem)->flags;
 805                 if (mem_flags & ACPI_NFIT_MEM_ARMED)
 806                         flags |= NDD_UNARMED;
 807
 808                 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
 809                 if (rc)
 810                         continue;
 811
 812                 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
 813                                 acpi_nfit_dimm_attribute_groups,
 814                                 flags, &nfit_mem->dsm_mask);
 815                 if (!nvdimm)
 816                         return -ENOMEM;
 817
 818                 nfit_mem->nvdimm = nvdimm;
 819                 dimm_count++;
 820
 821                 if ((mem_flags & ACPI_NFIT_MEM_FAILED_MASK) == 0)
 822                         continue;
 823
 824                 dev_info(acpi_desc->dev, "%s: failed: %s%s%s%s\n",
 825                                 nvdimm_name(nvdimm),
 826                         mem_flags & ACPI_NFIT_MEM_SAVE_FAILED ? "save " : "",
 827                         mem_flags & ACPI_NFIT_MEM_RESTORE_FAILED ? "restore " : "",
 828                         mem_flags & ACPI_NFIT_MEM_FLUSH_FAILED ? "flush " : "",
 829                         mem_flags & ACPI_NFIT_MEM_ARMED ? "arm " : "");
 830
 831         }
 832
 833         return nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
 834 }
 835
 836 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
 837 {
 838         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
 839         const u8 *uuid = to_nfit_uuid(NFIT_DEV_BUS);
 840         struct acpi_device *adev;
 841         int i;
 842
 843         adev = to_acpi_dev(acpi_desc);
 844         if (!adev)
 845                 return;
 846
 847         for (i = ND_CMD_ARS_CAP; i <= ND_CMD_ARS_STATUS; i++)
 848                 if (acpi_check_dsm(adev->handle, uuid, 1, 1ULL << i))
 849                         set_bit(i, &nd_desc->dsm_mask);
 850 }
 851
 852 static ssize_t range_index_show(struct device *dev,
 853                 struct device_attribute *attr, char *buf)
 854 {
 855         struct nd_region *nd_region = to_nd_region(dev);
 856         struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
 857
 858         return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
 859 }
 860 static DEVICE_ATTR_RO(range_index);
 861
 862 static struct attribute *acpi_nfit_region_attributes[] = {
 863         &dev_attr_range_index.attr,
 864         NULL,
 865 };
 866
 867 static struct attribute_group acpi_nfit_region_attribute_group = {
 868         .name = "nfit",
 869         .attrs = acpi_nfit_region_attributes,
 870 };
 871
 872 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
 873         &nd_region_attribute_group,
 874         &nd_mapping_attribute_group,
 875         &nd_device_attribute_group,
 876         &acpi_nfit_region_attribute_group,
 877         NULL,
 878 };
 879
 880 /* enough info to uniquely specify an interleave set */
 881 struct nfit_set_info {
 882         struct nfit_set_info_map {
 883                 u64 region_offset;
 884                 u32 serial_number;
 885                 u32 pad;
 886         } mapping[0];
 887 };
 888
 889 static size_t sizeof_nfit_set_info(int num_mappings)
 890 {
 891         return sizeof(struct nfit_set_info)
 892                 + num_mappings * sizeof(struct nfit_set_info_map);
 893 }
 894
 895 static int cmp_map(const void *m0, const void *m1)
 896 {
 897         const struct nfit_set_info_map *map0 = m0;
 898         const struct nfit_set_info_map *map1 = m1;
 899
 900         return memcmp(&map0->region_offset, &map1->region_offset,
 901                         sizeof(u64));
 902 }
 903
 904 /* Retrieve the nth entry referencing this spa */
 905 static struct acpi_nfit_memory_map *memdev_from_spa(
 906                 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
 907 {
 908         struct nfit_memdev *nfit_memdev;
 909
 910         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
 911                 if (nfit_memdev->memdev->range_index == range_index)
 912                         if (n-- == 0)
 913                                 return nfit_memdev->memdev;
 914         return NULL;
 915 }
 916
 917 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
 918                 struct nd_region_desc *ndr_desc,
 919                 struct acpi_nfit_system_address *spa)
 920 {
 921         int i, spa_type = nfit_spa_type(spa);
 922         struct device *dev = acpi_desc->dev;
 923         struct nd_interleave_set *nd_set;
 924         u16 nr = ndr_desc->num_mappings;
 925         struct nfit_set_info *info;
 926
 927         if (spa_type == NFIT_SPA_PM || spa_type == NFIT_SPA_VOLATILE)
 928                 /* pass */;
 929         else
 930                 return 0;
 931
 932         nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
 933         if (!nd_set)
 934                 return -ENOMEM;
 935
 936         info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
 937         if (!info)
 938                 return -ENOMEM;
 939         for (i = 0; i < nr; i++) {
 940                 struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
 941                 struct nfit_set_info_map *map = &info->mapping[i];
 942                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
 943                 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 944                 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
 945                                 spa->range_index, i);
 946
 947                 if (!memdev || !nfit_mem->dcr) {
 948                         dev_err(dev, "%s: failed to find DCR\n", __func__);
 949                         return -ENODEV;
 950                 }
 951
 952                 map->region_offset = memdev->region_offset;
 953                 map->serial_number = nfit_mem->dcr->serial_number;
 954         }
 955
 956         sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
 957                         cmp_map, NULL);
 958         nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
 959         ndr_desc->nd_set = nd_set;
 960         devm_kfree(dev, info);
 961
 962         return 0;
 963 }
 964
 965 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
 966 {
 967         struct acpi_nfit_interleave *idt = mmio->idt;
 968         u32 sub_line_offset, line_index, line_offset;
 969         u64 line_no, table_skip_count, table_offset;
 970
 971         line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
 972         table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
 973         line_offset = idt->line_offset[line_index]
 974                 * mmio->line_size;
 975         table_offset = table_skip_count * mmio->table_size;
 976
 977         return mmio->base_offset + line_offset + table_offset + sub_line_offset;
 978 }
 979
 980 static u64 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
 981 {
 982         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
 983         u64 offset = nfit_blk->stat_offset + mmio->size * bw;
 984
 985         if (mmio->num_lines)
 986                 offset = to_interleave_offset(offset, mmio);
 987
 988         return readq(mmio->base + offset);
 989 }
 990
 991 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
 992                 resource_size_t dpa, unsigned int len, unsigned int write)
 993 {
 994         u64 cmd, offset;
 995         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
 996
 997         enum {
 998                 BCW_OFFSET_MASK = (1ULL << 48)-1,
 999                 BCW_LEN_SHIFT = 48,
1000                 BCW_LEN_MASK = (1ULL << 8) - 1,
1001                 BCW_CMD_SHIFT = 56,
1002         };
1003
1004         cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
1005         len = len >> L1_CACHE_SHIFT;
1006         cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
1007         cmd |= ((u64) write) << BCW_CMD_SHIFT;
1008
1009         offset = nfit_blk->cmd_offset + mmio->size * bw;
1010         if (mmio->num_lines)
1011                 offset = to_interleave_offset(offset, mmio);
1012
1013         writeq(cmd, mmio->base + offset);
1014         /* FIXME: conditionally perform read-back if mandated by firmware */
1015 }
1016
1017 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
1018                 resource_size_t dpa, void *iobuf, size_t len, int rw,
1019                 unsigned int lane)
1020 {
1021         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1022         unsigned int copied = 0;
1023         u64 base_offset;
1024         int rc;
1025
1026         base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
1027                 + lane * mmio->size;
1028         /* TODO: non-temporal access, flush hints, cache management etc... */
1029         write_blk_ctl(nfit_blk, lane, dpa, len, rw);
1030         while (len) {
1031                 unsigned int c;
1032                 u64 offset;
1033
1034                 if (mmio->num_lines) {
1035                         u32 line_offset;
1036
1037                         offset = to_interleave_offset(base_offset + copied,
1038                                         mmio);
1039                         div_u64_rem(offset, mmio->line_size, &line_offset);
1040                         c = min_t(size_t, len, mmio->line_size - line_offset);
1041                 } else {
1042                         offset = base_offset + nfit_blk->bdw_offset;
1043                         c = len;
1044                 }
1045
1046                 if (rw)
1047                         memcpy(mmio->aperture + offset, iobuf + copied, c);
1048                 else
1049                         memcpy(iobuf + copied, mmio->aperture + offset, c);
1050
1051                 copied += c;
1052                 len -= c;
1053         }
1054         rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
1055         return rc;
1056 }
1057
1058 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
1059                 resource_size_t dpa, void *iobuf, u64 len, int rw)
1060 {
1061         struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1062         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1063         struct nd_region *nd_region = nfit_blk->nd_region;
1064         unsigned int lane, copied = 0;
1065         int rc = 0;
1066
1067         lane = nd_region_acquire_lane(nd_region);
1068         while (len) {
1069                 u64 c = min(len, mmio->size);
1070
1071                 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
1072                                 iobuf + copied, c, rw, lane);
1073                 if (rc)
1074                         break;
1075
1076                 copied += c;
1077                 len -= c;
1078         }
1079         nd_region_release_lane(nd_region, lane);
1080
1081         return rc;
1082 }
1083
1084 static void nfit_spa_mapping_release(struct kref *kref)
1085 {
1086         struct nfit_spa_mapping *spa_map = to_spa_map(kref);
1087         struct acpi_nfit_system_address *spa = spa_map->spa;
1088         struct acpi_nfit_desc *acpi_desc = spa_map->acpi_desc;
1089
1090         WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1091         dev_dbg(acpi_desc->dev, "%s: SPA%d\n", __func__, spa->range_index);
1092         iounmap(spa_map->iomem);
1093         release_mem_region(spa->address, spa->length);
1094         list_del(&spa_map->list);
1095         kfree(spa_map);
1096 }
1097
1098 static struct nfit_spa_mapping *find_spa_mapping(
1099                 struct acpi_nfit_desc *acpi_desc,
1100                 struct acpi_nfit_system_address *spa)
1101 {
1102         struct nfit_spa_mapping *spa_map;
1103
1104         WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1105         list_for_each_entry(spa_map, &acpi_desc->spa_maps, list)
1106                 if (spa_map->spa == spa)
1107                         return spa_map;
1108
1109         return NULL;
1110 }
1111
1112 static void nfit_spa_unmap(struct acpi_nfit_desc *acpi_desc,
1113                 struct acpi_nfit_system_address *spa)
1114 {
1115         struct nfit_spa_mapping *spa_map;
1116
1117         mutex_lock(&acpi_desc->spa_map_mutex);
1118         spa_map = find_spa_mapping(acpi_desc, spa);
1119
1120         if (spa_map)
1121                 kref_put(&spa_map->kref, nfit_spa_mapping_release);
1122         mutex_unlock(&acpi_desc->spa_map_mutex);
1123 }
1124
1125 static void __iomem *__nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
1126                 struct acpi_nfit_system_address *spa)
1127 {
1128         resource_size_t start = spa->address;
1129         resource_size_t n = spa->length;
1130         struct nfit_spa_mapping *spa_map;
1131         struct resource *res;
1132
1133         WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1134
1135         spa_map = find_spa_mapping(acpi_desc, spa);
1136         if (spa_map) {
1137                 kref_get(&spa_map->kref);
1138                 return spa_map->iomem;
1139         }
1140
1141         spa_map = kzalloc(sizeof(*spa_map), GFP_KERNEL);
1142         if (!spa_map)
1143                 return NULL;
1144
1145         INIT_LIST_HEAD(&spa_map->list);
1146         spa_map->spa = spa;
1147         kref_init(&spa_map->kref);
1148         spa_map->acpi_desc = acpi_desc;
1149
1150         res = request_mem_region(start, n, dev_name(acpi_desc->dev));
1151         if (!res)
1152                 goto err_mem;
1153
1154         /* TODO: cacheability based on the spa type */
1155         spa_map->iomem = ioremap_nocache(start, n);
1156         if (!spa_map->iomem)
1157                 goto err_map;
1158
1159         list_add_tail(&spa_map->list, &acpi_desc->spa_maps);
1160         return spa_map->iomem;
1161
1162  err_map:
1163         release_mem_region(start, n);
1164  err_mem:
1165         kfree(spa_map);
1166         return NULL;
1167 }
1168
1169 /**
1170  * nfit_spa_map - interleave-aware managed-mappings of acpi_nfit_system_address ranges
1171  * @nvdimm_bus: NFIT-bus that provided the spa table entry
1172  * @nfit_spa: spa table to map
1173  *
1174  * In the case where block-data-window apertures and
1175  * dimm-control-regions are interleaved they will end up sharing a
1176  * single request_mem_region() + ioremap() for the address range.  In
1177  * the style of devm nfit_spa_map() mappings are automatically dropped
1178  * when all region devices referencing the same mapping are disabled /
1179  * unbound.
1180  */
1181 static void __iomem *nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
1182                 struct acpi_nfit_system_address *spa)
1183 {
1184         void __iomem *iomem;
1185
1186         mutex_lock(&acpi_desc->spa_map_mutex);
1187         iomem = __nfit_spa_map(acpi_desc, spa);
1188         mutex_unlock(&acpi_desc->spa_map_mutex);
1189
1190         return iomem;
1191 }
1192
1193 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
1194                 struct acpi_nfit_interleave *idt, u16 interleave_ways)
1195 {
1196         if (idt) {
1197                 mmio->num_lines = idt->line_count;
1198                 mmio->line_size = idt->line_size;
1199                 if (interleave_ways == 0)
1200                         return -ENXIO;
1201                 mmio->table_size = mmio->num_lines * interleave_ways
1202                         * mmio->line_size;
1203         }
1204
1205         return 0;
1206 }
1207
1208 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
1209                 struct device *dev)
1210 {
1211         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1212         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1213         struct nd_blk_region *ndbr = to_nd_blk_region(dev);
1214         struct nfit_blk_mmio *mmio;
1215         struct nfit_blk *nfit_blk;
1216         struct nfit_mem *nfit_mem;
1217         struct nvdimm *nvdimm;
1218         int rc;
1219
1220         nvdimm = nd_blk_region_to_dimm(ndbr);
1221         nfit_mem = nvdimm_provider_data(nvdimm);
1222         if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
1223                 dev_dbg(dev, "%s: missing%s%s%s\n", __func__,
1224                                 nfit_mem ? "" : " nfit_mem",
1225                                 nfit_mem->dcr ? "" : " dcr",
1226                                 nfit_mem->bdw ? "" : " bdw");
1227                 return -ENXIO;
1228         }
1229
1230         nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
1231         if (!nfit_blk)
1232                 return -ENOMEM;
1233         nd_blk_region_set_provider_data(ndbr, nfit_blk);
1234         nfit_blk->nd_region = to_nd_region(dev);
1235
1236         /* map block aperture memory */
1237         nfit_blk->bdw_offset = nfit_mem->bdw->offset;
1238         mmio = &nfit_blk->mmio[BDW];
1239         mmio->base = nfit_spa_map(acpi_desc, nfit_mem->spa_bdw);
1240         if (!mmio->base) {
1241                 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__,
1242                                 nvdimm_name(nvdimm));
1243                 return -ENOMEM;
1244         }
1245         mmio->size = nfit_mem->bdw->size;
1246         mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
1247         mmio->idt = nfit_mem->idt_bdw;
1248         mmio->spa = nfit_mem->spa_bdw;
1249         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
1250                         nfit_mem->memdev_bdw->interleave_ways);
1251         if (rc) {
1252                 dev_dbg(dev, "%s: %s failed to init bdw interleave\n",
1253                                 __func__, nvdimm_name(nvdimm));
1254                 return rc;
1255         }
1256
1257         /* map block control memory */
1258         nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
1259         nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
1260         mmio = &nfit_blk->mmio[DCR];
1261         mmio->base = nfit_spa_map(acpi_desc, nfit_mem->spa_dcr);
1262         if (!mmio->base) {
1263                 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__,
1264                                 nvdimm_name(nvdimm));
1265                 return -ENOMEM;
1266         }
1267         mmio->size = nfit_mem->dcr->window_size;
1268         mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
1269         mmio->idt = nfit_mem->idt_dcr;
1270         mmio->spa = nfit_mem->spa_dcr;
1271         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
1272                         nfit_mem->memdev_dcr->interleave_ways);
1273         if (rc) {
1274                 dev_dbg(dev, "%s: %s failed to init dcr interleave\n",
1275                                 __func__, nvdimm_name(nvdimm));
1276                 return rc;
1277         }
1278
1279         if (mmio->line_size == 0)
1280                 return 0;
1281
1282         if ((u32) nfit_blk->cmd_offset % mmio->line_size
1283                         + 8 > mmio->line_size) {
1284                 dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
1285                 return -ENXIO;
1286         } else if ((u32) nfit_blk->stat_offset % mmio->line_size
1287                         + 8 > mmio->line_size) {
1288                 dev_dbg(dev, "stat_offset crosses interleave boundary\n");
1289                 return -ENXIO;
1290         }
1291
1292         return 0;
1293 }
1294
1295 static void acpi_nfit_blk_region_disable(struct nvdimm_bus *nvdimm_bus,
1296                 struct device *dev)
1297 {
1298         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1299         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1300         struct nd_blk_region *ndbr = to_nd_blk_region(dev);
1301         struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1302         int i;
1303
1304         if (!nfit_blk)
1305                 return; /* never enabled */
1306
1307         /* auto-free BLK spa mappings */
1308         for (i = 0; i < 2; i++) {
1309                 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[i];
1310
1311                 if (mmio->base)
1312                         nfit_spa_unmap(acpi_desc, mmio->spa);
1313         }
1314         nd_blk_region_set_provider_data(ndbr, NULL);
1315         /* devm will free nfit_blk */
1316 }
1317
1318 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
1319                 struct nd_mapping *nd_mapping, struct nd_region_desc *ndr_desc,
1320                 struct acpi_nfit_memory_map *memdev,
1321                 struct acpi_nfit_system_address *spa)
1322 {
1323         struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
1324                         memdev->device_handle);
1325         struct nd_blk_region_desc *ndbr_desc;
1326         struct nfit_mem *nfit_mem;
1327         int blk_valid = 0;
1328
1329         if (!nvdimm) {
1330                 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
1331                                 spa->range_index, memdev->device_handle);
1332                 return -ENODEV;
1333         }
1334
1335         nd_mapping->nvdimm = nvdimm;
1336         switch (nfit_spa_type(spa)) {
1337         case NFIT_SPA_PM:
1338         case NFIT_SPA_VOLATILE:
1339                 nd_mapping->start = memdev->address;
1340                 nd_mapping->size = memdev->region_size;
1341                 break;
1342         case NFIT_SPA_DCR:
1343                 nfit_mem = nvdimm_provider_data(nvdimm);
1344                 if (!nfit_mem || !nfit_mem->bdw) {
1345                         dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
1346                                         spa->range_index, nvdimm_name(nvdimm));
1347                 } else {
1348                         nd_mapping->size = nfit_mem->bdw->capacity;
1349                         nd_mapping->start = nfit_mem->bdw->start_address;
1350                         ndr_desc->num_lanes = nfit_mem->bdw->windows;
1351                         blk_valid = 1;
1352                 }
1353
1354                 ndr_desc->nd_mapping = nd_mapping;
1355                 ndr_desc->num_mappings = blk_valid;
1356                 ndbr_desc = to_blk_region_desc(ndr_desc);
1357                 ndbr_desc->enable = acpi_nfit_blk_region_enable;
1358                 ndbr_desc->disable = acpi_nfit_blk_region_disable;
1359                 ndbr_desc->do_io = acpi_desc->blk_do_io;
1360                 if (!nvdimm_blk_region_create(acpi_desc->nvdimm_bus, ndr_desc))
1361                         return -ENOMEM;
1362                 break;
1363         }
1364
1365         return 0;
1366 }
1367
1368 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
1369                 struct nfit_spa *nfit_spa)
1370 {
1371         static struct nd_mapping nd_mappings[ND_MAX_MAPPINGS];
1372         struct acpi_nfit_system_address *spa = nfit_spa->spa;
1373         struct nd_blk_region_desc ndbr_desc;
1374         struct nd_region_desc *ndr_desc;
1375         struct nfit_memdev *nfit_memdev;
1376         struct nvdimm_bus *nvdimm_bus;
1377         struct resource res;
1378         int count = 0, rc;
1379
1380         if (spa->range_index == 0) {
1381                 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n",
1382                                 __func__);
1383                 return 0;
1384         }
1385
1386         memset(&res, 0, sizeof(res));
1387         memset(&nd_mappings, 0, sizeof(nd_mappings));
1388         memset(&ndbr_desc, 0, sizeof(ndbr_desc));
1389         res.start = spa->address;
1390         res.end = res.start + spa->length - 1;
1391         ndr_desc = &ndbr_desc.ndr_desc;
1392         ndr_desc->res = &res;
1393         ndr_desc->provider_data = nfit_spa;
1394         ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
1395         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1396                 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1397                 struct nd_mapping *nd_mapping;
1398
1399                 if (memdev->range_index != spa->range_index)
1400                         continue;
1401                 if (count >= ND_MAX_MAPPINGS) {
1402                         dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
1403                                         spa->range_index, ND_MAX_MAPPINGS);
1404                         return -ENXIO;
1405                 }
1406                 nd_mapping = &nd_mappings[count++];
1407                 rc = acpi_nfit_init_mapping(acpi_desc, nd_mapping, ndr_desc,
1408                                 memdev, spa);
1409                 if (rc)
1410                         return rc;
1411         }
1412
1413         ndr_desc->nd_mapping = nd_mappings;
1414         ndr_desc->num_mappings = count;
1415         rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
1416         if (rc)
1417                 return rc;
1418
1419         nvdimm_bus = acpi_desc->nvdimm_bus;
1420         if (nfit_spa_type(spa) == NFIT_SPA_PM) {
1421                 if (!nvdimm_pmem_region_create(nvdimm_bus, ndr_desc))
1422                         return -ENOMEM;
1423         } else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) {
1424                 if (!nvdimm_volatile_region_create(nvdimm_bus, ndr_desc))
1425                         return -ENOMEM;
1426         }
1427         return 0;
1428 }
1429
1430 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
1431 {
1432         struct nfit_spa *nfit_spa;
1433
1434         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1435                 int rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
1436
1437                 if (rc)
1438                         return rc;
1439         }
1440         return 0;
1441 }
1442
1443 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, acpi_size sz)
1444 {
1445         struct device *dev = acpi_desc->dev;
1446         const void *end;
1447         u8 *data;
1448         int rc;
1449
1450         INIT_LIST_HEAD(&acpi_desc->spa_maps);
1451         INIT_LIST_HEAD(&acpi_desc->spas);
1452         INIT_LIST_HEAD(&acpi_desc->dcrs);
1453         INIT_LIST_HEAD(&acpi_desc->bdws);
1454         INIT_LIST_HEAD(&acpi_desc->idts);
1455         INIT_LIST_HEAD(&acpi_desc->memdevs);
1456         INIT_LIST_HEAD(&acpi_desc->dimms);
1457         mutex_init(&acpi_desc->spa_map_mutex);
1458
1459         data = (u8 *) acpi_desc->nfit;
1460         end = data + sz;
1461         data += sizeof(struct acpi_table_nfit);
1462         while (!IS_ERR_OR_NULL(data))
1463                 data = add_table(acpi_desc, data, end);
1464
1465         if (IS_ERR(data)) {
1466                 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__,
1467                                 PTR_ERR(data));
1468                 return PTR_ERR(data);
1469         }
1470
1471         if (nfit_mem_init(acpi_desc) != 0)
1472                 return -ENOMEM;
1473
1474         acpi_nfit_init_dsms(acpi_desc);
1475
1476         rc = acpi_nfit_register_dimms(acpi_desc);
1477         if (rc)
1478                 return rc;
1479
1480         return acpi_nfit_register_regions(acpi_desc);
1481 }
1482 EXPORT_SYMBOL_GPL(acpi_nfit_init);
1483
1484 static int acpi_nfit_add(struct acpi_device *adev)
1485 {
1486         struct nvdimm_bus_descriptor *nd_desc;
1487         struct acpi_nfit_desc *acpi_desc;
1488         struct device *dev = &adev->dev;
1489         struct acpi_table_header *tbl;
1490         acpi_status status = AE_OK;
1491         acpi_size sz;
1492         int rc;
1493
1494         status = acpi_get_table_with_size("NFIT", 0, &tbl, &sz);
1495         if (ACPI_FAILURE(status)) {
1496                 dev_err(dev, "failed to find NFIT\n");
1497                 return -ENXIO;
1498         }
1499
1500         acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
1501         if (!acpi_desc)
1502                 return -ENOMEM;
1503
1504         dev_set_drvdata(dev, acpi_desc);
1505         acpi_desc->dev = dev;
1506         acpi_desc->nfit = (struct acpi_table_nfit *) tbl;
1507         acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
1508         nd_desc = &acpi_desc->nd_desc;
1509         nd_desc->provider_name = "ACPI.NFIT";
1510         nd_desc->ndctl = acpi_nfit_ctl;
1511         nd_desc->attr_groups = acpi_nfit_attribute_groups;
1512
1513         acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, nd_desc);
1514         if (!acpi_desc->nvdimm_bus)
1515                 return -ENXIO;
1516
1517         rc = acpi_nfit_init(acpi_desc, sz);
1518         if (rc) {
1519                 nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
1520                 return rc;
1521         }
1522         return 0;
1523 }
1524
1525 static int acpi_nfit_remove(struct acpi_device *adev)
1526 {
1527         struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev);
1528
1529         nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
1530         return 0;
1531 }
1532
1533 static const struct acpi_device_id acpi_nfit_ids[] = {
1534         { "ACPI0012", 0 },
1535         { "", 0 },
1536 };
1537 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
1538
1539 static struct acpi_driver acpi_nfit_driver = {
1540         .name = KBUILD_MODNAME,
1541         .ids = acpi_nfit_ids,
1542         .ops = {
1543                 .add = acpi_nfit_add,
1544                 .remove = acpi_nfit_remove,
1545         },
1546 };
1547
1548 static __init int nfit_init(void)
1549 {
1550         BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
1551         BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
1552         BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
1553         BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
1554         BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
1555         BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
1556         BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
1557
1558         acpi_str_to_uuid(UUID_VOLATILE_MEMORY, nfit_uuid[NFIT_SPA_VOLATILE]);
1559         acpi_str_to_uuid(UUID_PERSISTENT_MEMORY, nfit_uuid[NFIT_SPA_PM]);
1560         acpi_str_to_uuid(UUID_CONTROL_REGION, nfit_uuid[NFIT_SPA_DCR]);
1561         acpi_str_to_uuid(UUID_DATA_REGION, nfit_uuid[NFIT_SPA_BDW]);
1562         acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_VDISK]);
1563         acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD, nfit_uuid[NFIT_SPA_VCD]);
1564         acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_PDISK]);
1565         acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD, nfit_uuid[NFIT_SPA_PCD]);
1566         acpi_str_to_uuid(UUID_NFIT_BUS, nfit_uuid[NFIT_DEV_BUS]);
1567         acpi_str_to_uuid(UUID_NFIT_DIMM, nfit_uuid[NFIT_DEV_DIMM]);
1568
1569         return acpi_bus_register_driver(&acpi_nfit_driver);
1570 }
1571
1572 static __exit void nfit_exit(void)
1573 {
1574         acpi_bus_unregister_driver(&acpi_nfit_driver);
1575 }
1576
1577 module_init(nfit_init);
1578 module_exit(nfit_exit);
1579 MODULE_LICENSE("GPL v2");
1580 MODULE_AUTHOR("Intel Corporation");